SherbyNet - /page/wikibook/entry.mu

Micron Document

<!DOCTYPE html>

<title>Weight initialization</title>

</head>

<span id="openzim-page-title" class="mw-page-title-main"><span class="mw-page-title-main">Weight initialization</span></span>

</h1>

</header>

/* start https://en.wikipedia.org/ */

.mw-parser-output .hlist dl,.mw-parser-output .hlist ol,.mw-parser-output .hlist ul{margin:0;padding:0}.mw-parser-output .hlist dd,.mw-parser-output .hlist dt,.mw-parser-output .hlist li{margin:0;display:inline}.mw-parser-output .hlist.inline,.mw-parser-output .hlist.inline dl,.mw-parser-output .hlist.inline ol,.mw-parser-output .hlist.inline ul,.mw-parser-output .hlist dl dl,.mw-parser-output .hlist dl ol,.mw-parser-output .hlist dl ul,.mw-parser-output .hlist ol dl,.mw-parser-output .hlist ol ol,.mw-parser-output .hlist ol ul,.mw-parser-output .hlist ul dl,.mw-parser-output .hlist ul ol,.mw-parser-output .hlist ul ul{display:inline}.mw-parser-output .hlist .mw-empty-li{display:none}.mw-parser-output .hlist dt::after{content:": "}.mw-parser-output .hlist dd::after,.mw-parser-output .hlist li::after{content:" · ";font-weight:bold}.mw-parser-output .hlist dd:last-child::after,.mw-parser-output .hlist dt:last-child::after,.mw-parser-output .hlist li:last-child::after{content:none}.mw-parser-output .hlist dd dd:first-child::before,.mw-parser-output .hlist dd dt:first-child::before,.mw-parser-output .hlist dd li:first-child::before,.mw-parser-output .hlist dt dd:first-child::before,.mw-parser-output .hlist dt dt:first-child::before,.mw-parser-output .hlist dt li:first-child::before,.mw-parser-output .hlist li dd:first-child::before,.mw-parser-output .hlist li dt:first-child::before,.mw-parser-output .hlist li li:first-child::before{content:" (";font-weight:normal}.mw-parser-output .hlist dd dd:last-child::after,.mw-parser-output .hlist dd dt:last-child::after,.mw-parser-output .hlist dd li:last-child::after,.mw-parser-output .hlist dt dd:last-child::after,.mw-parser-output .hlist dt dt:last-child::after,.mw-parser-output .hlist dt li:last-child::after,.mw-parser-output .hlist li dd:last-child::after,.mw-parser-output .hlist li dt:last-child::after,.mw-parser-output .hlist li li:last-child::after{content:")";font-weight:normal}.mw-parser-output .hlist ol{counter-reset:listitem}.mw-parser-output .hlist ol>li{counter-increment:listitem}.mw-parser-output .hlist ol>li::before{content:" "counter(listitem)"\a0 "}.mw-parser-output .hlist dd ol>li:first-child::before,.mw-parser-output .hlist dt ol>li:first-child::before,.mw-parser-output .hlist li ol>li:first-child::before{content:" ("counter(listitem)"\a0 "}

/* end https://en.wikipedia.org/ */

</style><style data-mw-deduplicate="TemplateStyles:r1246091330">

/* start https://en.wikipedia.org/ */

.mw-parser-output .sidebar{width:22em;float:right;clear:right;margin:0.5em 0 1em 1em;background:var(--background-color-neutral-subtle,#f8f9fa);border:1px solid var(--border-color-base,#a2a9b1);padding:0.2em;text-align:center;line-height:1.4em;font-size:88%;border-collapse:collapse;display:table}body.skin-minerva .mw-parser-output .sidebar{display:table!important;float:right!important;margin:0.5em 0 1em 1em!important}.mw-parser-output .sidebar-subgroup{width:100%;margin:0;border-spacing:0}.mw-parser-output .sidebar-left{float:left;clear:left;margin:0.5em 1em 1em 0}.mw-parser-output .sidebar-none{float:none;clear:both;margin:0.5em 1em 1em 0}.mw-parser-output .sidebar-outer-title{padding:0 0.4em 0.2em;font-size:125%;line-height:1.2em;font-weight:bold}.mw-parser-output .sidebar-top-image{padding:0.4em}.mw-parser-output .sidebar-top-caption,.mw-parser-output .sidebar-pretitle-with-top-image,.mw-parser-output .sidebar-caption{padding:0.2em 0.4em 0;line-height:1.2em}.mw-parser-output .sidebar-pretitle{padding:0.4em 0.4em 0;line-height:1.2em}.mw-parser-output .sidebar-title,.mw-parser-output .sidebar-title-with-pretitle{padding:0.2em 0.8em;font-size:145%;line-height:1.2em}.mw-parser-output .sidebar-title-with-pretitle{padding:0.1em 0.4em}.mw-parser-output .sidebar-image{padding:0.2em 0.4em 0.4em}.mw-parser-output .sidebar-heading{padding:0.1em 0.4em}.mw-parser-output .sidebar-content{padding:0 0.5em 0.4em}.mw-parser-output .sidebar-content-with-subgroup{padding:0.1em 0.4em 0.2em}.mw-parser-output .sidebar-above,.mw-parser-output .sidebar-below{padding:0.3em 0.8em;font-weight:bold}.mw-parser-output .sidebar-collapse .sidebar-above,.mw-parser-output .sidebar-collapse .sidebar-below{border-top:1px solid #aaa;border-bottom:1px solid #aaa}.mw-parser-output .sidebar-navbar{text-align:right;font-size:115%;padding:0 0.4em 0.4em}.mw-parser-output .sidebar-list-title{padding:0 0.4em;text-align:left;font-weight:bold;line-height:1.6em;font-size:105%}.mw-parser-output .sidebar-list-title-c{padding:0 0.4em;text-align:center;margin:0 3.3em}@media(max-width:640px){body.mediawiki .mw-parser-output .sidebar{width:100%!important;clear:both;float:none!important;margin-left:0!important;margin-right:0!important}}body.skin--responsive .mw-parser-output .sidebar a>img{max-width:none!important}@media screen{html.skin-theme-clientpref-night .mw-parser-output .sidebar:not(.notheme) .sidebar-list-title,html.skin-theme-clientpref-night .mw-parser-output .sidebar:not(.notheme) .sidebar-title-with-pretitle{background:transparent!important}html.skin-theme-clientpref-night .mw-parser-output .sidebar:not(.notheme) .sidebar-title-with-pretitle a{color:var(--color-progressive)!important}}@media screen and (prefers-color-scheme:dark){html.skin-theme-clientpref-os .mw-parser-output .sidebar:not(.notheme) .sidebar-list-title,html.skin-theme-clientpref-os .mw-parser-output .sidebar:not(.notheme) .sidebar-title-with-pretitle{background:transparent!important}html.skin-theme-clientpref-os .mw-parser-output .sidebar:not(.notheme) .sidebar-title-with-pretitle a{color:var(--color-progressive)!important}}@media print{body.ns-0 .mw-parser-output .sidebar{display:none!important}}

/* end https://en.wikipedia.org/ */

</style><style data-mw-deduplicate="TemplateStyles:r886047488">

/* start https://en.wikipedia.org/ */

.mw-parser-output .nobold{font-weight:normal}

/* end https://en.wikipedia.org/ */

</style><table class="sidebar sidebar-collapse nomobile nowraplinks"><tbody><tr><td class="sidebar-pretitle">Part of a series on</td></tr><tr><th class="sidebar-title-with-pretitle"><a href="Machine_learning" title="Machine learning">Machine learning</a><br>and <a href="Data_mining" title="Data mining">data mining</a></th></tr><tr><td class="sidebar-content">

<ul><li><a href="Supervised_learning" title="Supervised learning">Supervised learning</a></li>

<li><a href="Unsupervised_learning" title="Unsupervised learning">Unsupervised learning</a></li>

<li><a href="Semi-supervised_learning" class="mw-redirect" title="Semi-supervised learning">Semi-supervised learning</a></li>

<li><a href="Self-supervised_learning" title="Self-supervised learning">Self-supervised learning</a></li>

<li><a href="Reinforcement_learning" title="Reinforcement learning">Reinforcement learning</a></li>

<li><a href="Meta-learning_(computer_science)" title="Meta-learning (computer science)">Meta-learning</a></li>

<li><a href="Online_machine_learning" title="Online machine learning">Online learning</a></li>

<li><a href="Batch_learning" class="mw-redirect" title="Batch learning">Batch learning</a></li>

<li><a href="Curriculum_learning" title="Curriculum learning">Curriculum learning</a></li>

<li><a href="Rule-based_machine_learning" title="Rule-based machine learning">Rule-based learning</a></li>

<li><a href="Neuro-symbolic_AI" title="Neuro-symbolic AI">Neuro-symbolic AI</a></li>

<li><a href="Neuromorphic_engineering" class="mw-redirect" title="Neuromorphic engineering">Neuromorphic engineering</a></li>

<li><a href="Quantum_machine_learning" title="Quantum machine learning">Quantum machine learning</a></li></ul></div></div></td>

</tr><tr><td class="sidebar-content">

<ul><li><a href="Statistical_classification" title="Statistical classification">Classification</a></li>

<li><a href="Generative_model" title="Generative model">Generative modeling</a></li>

<li><a href="Regression_analysis" title="Regression analysis">Regression</a></li>

<li><a href="Cluster_analysis" title="Cluster analysis">Clustering</a></li>

<li><a href="Dimensionality_reduction" title="Dimensionality reduction">Dimensionality reduction</a></li>

<li><a href="Density_estimation" title="Density estimation">Density estimation</a></li>

<li><a href="Anomaly_detection" title="Anomaly detection">Anomaly detection</a></li>

<li><a href="Data_cleaning" class="mw-redirect" title="Data cleaning">Data cleaning</a></li>

<li><a href="Automated_machine_learning" title="Automated machine learning">AutoML</a></li>

<li><a href="Association_rule_learning" title="Association rule learning">Association rules</a></li>

<li><a href="Semantic_analysis_(machine_learning)" title="Semantic analysis (machine learning)">Semantic analysis</a></li>

<li><a href="Structured_prediction" title="Structured prediction">Structured prediction</a></li>

<li><a href="Feature_engineering" title="Feature engineering">Feature engineering</a></li>

<li><a href="Feature_learning" title="Feature learning">Feature learning</a></li>

<li><a href="Learning_to_rank" title="Learning to rank">Learning to rank</a></li>

<li><a href="Grammar_induction" title="Grammar induction">Grammar induction</a></li>

<li><a href="Ontology_learning" title="Ontology learning">Ontology learning</a></li>

<li><a href="Multimodal_learning" title="Multimodal learning">Multimodal learning</a></li></ul></div></div></td>

</tr><tr><td class="sidebar-content">

<div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)"><div style="display: inline-block; line-height: 1.2em; padding: .1em 0;"><a href="Supervised_learning" title="Supervised learning">Supervised learning</a><br><span class="nobold"><span style="font-size: 85%;">(<b><a href="Statistical_classification" title="Statistical classification">classification</a></b> • <b><a href="Regression_analysis" title="Regression analysis">regression</a></b>)</span></span> </div></div><div class="sidebar-list-content mw-collapsible-content hlist">

<ul><li><a href="Apprenticeship_learning" title="Apprenticeship learning">Apprenticeship learning</a></li>

<li><a href="Decision_tree_learning" title="Decision tree learning">Decision trees</a></li>

<li><a href="Ensemble_learning" title="Ensemble learning">Ensembles</a>

<ul><li><a href="Bootstrap_aggregating" title="Bootstrap aggregating">Bagging</a></li>

<li><a href="Boosting_(machine_learning)" title="Boosting (machine learning)">Boosting</a></li>

<li><a href="Random_forest" title="Random forest">Random forest</a></li></ul></li>

<li><a href="Linear_regression" title="Linear regression">Linear regression</a></li>

<li><a href="Naive_Bayes_classifier" title="Naive Bayes classifier">Naive Bayes</a></li>

<li><a href="Artificial_neural_network" class="mw-redirect" title="Artificial neural network">Artificial neural networks</a></li>

<li><a href="Logistic_regression" title="Logistic regression">Logistic regression</a></li>

<li><a href="Perceptron" title="Perceptron">Perceptron</a></li>

<li><a href="Relevance_vector_machine" title="Relevance vector machine">Relevance vector machine (RVM)</a></li>

<li><a href="Support_vector_machine" title="Support vector machine">Support vector machine (SVM)</a></li></ul></div></div></td>

</tr><tr><td class="sidebar-content">

<div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)"><a href="Cluster_analysis" title="Cluster analysis">Clustering</a></div><div class="sidebar-list-content mw-collapsible-content hlist">

<ul><li><a href="BIRCH" title="BIRCH">BIRCH</a></li>

<li><a href="Hierarchical_clustering" title="Hierarchical clustering">Hierarchical</a></li>

<li><a href="K-means_clustering" title="K-means clustering"><i>k</i>-means</a></li>

<li><a href="Fuzzy_clustering" title="Fuzzy clustering">Fuzzy</a></li>

<li><a href="Expectation%E2%80%93maximization_algorithm" title="Expectation–maximization algorithm">Expectation–maximization (EM)</a></li>

<li><br><a href="DBSCAN" title="DBSCAN">DBSCAN</a></li>

<li><a href="OPTICS_algorithm" title="OPTICS algorithm">OPTICS</a></li>

<li><a href="Mean_shift" title="Mean shift">Mean shift</a></li></ul></div></div></td>

</tr><tr><td class="sidebar-content">

<div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)"><a href="Dimensionality_reduction" title="Dimensionality reduction">Dimensionality reduction</a></div><div class="sidebar-list-content mw-collapsible-content hlist">

<ul><li><a href="Factor_analysis" title="Factor analysis">Factor analysis</a></li>

</tr><tr><td class="sidebar-content">

<div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)"><a href="Structured_prediction" title="Structured prediction">Structured prediction</a></div><div class="sidebar-list-content mw-collapsible-content hlist">

<ul><li><a href="Graphical_model" title="Graphical model">Graphical models</a>

<ul><li><a href="Bayesian_network" title="Bayesian network">Bayes net</a></li>

<li><a href="Conditional_random_field" title="Conditional random field">Conditional random field</a></li>

<li><a href="Hidden_Markov_model" title="Hidden Markov model">Hidden Markov</a></li></ul></li></ul></div></div></td>

</tr><tr><td class="sidebar-content">

<div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)"><a href="Anomaly_detection" title="Anomaly detection">Anomaly detection</a></div><div class="sidebar-list-content mw-collapsible-content hlist">

<ul><li><a href="Random_sample_consensus" title="Random sample consensus">RANSAC</a></li>

<li><a href="Local_outlier_factor" title="Local outlier factor">Local outlier factor</a></li>

<li><a href="Isolation_forest" title="Isolation forest">Isolation forest</a></li></ul></div></div></td>

</tr><tr><td class="sidebar-content">

<div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)"><a href="Neural_network_(machine_learning)" title="Neural network (machine learning)">Neural networks</a></div><div class="sidebar-list-content mw-collapsible-content hlist">

<ul><li><a href="Autoencoder" title="Autoencoder">Autoencoder</a></li>

<li><a href="Deep_learning" title="Deep learning">Deep learning</a></li>

<li><a href="Feedforward_neural_network" title="Feedforward neural network">Feedforward neural network</a></li>

<li><a href="Recurrent_neural_network" title="Recurrent neural network">Recurrent neural network</a>

<li><a href="Reservoir_computing" title="Reservoir computing">reservoir computing</a></li></ul></li>

<li><a href="Boltzmann_machine" title="Boltzmann machine">Boltzmann machine</a>

<ul><li><a href="Restricted_Boltzmann_machine" title="Restricted Boltzmann machine">Restricted</a></li></ul></li>

<li><a href="Diffusion_model" title="Diffusion model">Diffusion model</a></li>

<li><a href="Convolutional_neural_network" title="Convolutional neural network">Convolutional neural network</a>

<li><a href="LeNet" title="LeNet">LeNet</a></li>

<li><a href="AlexNet" title="AlexNet">AlexNet</a></li>

<li><a href="DeepDream" title="DeepDream">DeepDream</a></li></ul></li>

<li><a href="Neural_field" title="Neural field">Neural field</a>

<ul><li><a href="Neural_radiance_field" title="Neural radiance field">Neural radiance field</a></li>

<li><a href="Physics-informed_neural_networks" title="Physics-informed neural networks">Physics-informed neural networks</a></li></ul></li>

<li><a href="Transformer_(deep_learning_architecture)" title="Transformer (deep learning architecture)">Transformer</a>

<ul><li><a href="Vision_transformer" title="Vision transformer">Vision</a></li></ul></li>

<li><a href="Mamba_(deep_learning_architecture)" title="Mamba (deep learning architecture)">Mamba</a></li>

<li><a href="Spiking_neural_network" title="Spiking neural network">Spiking neural network</a></li>

<li><a href="Memtransistor" title="Memtransistor">Memtransistor</a></li>

<li><a href="Electrochemical_RAM" title="Electrochemical RAM">Electrochemical RAM</a> (ECRAM)</li></ul></div></div></td>

</tr><tr><td class="sidebar-content">

<div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)"><a href="Reinforcement_learning" title="Reinforcement learning">Reinforcement learning</a></div><div class="sidebar-list-content mw-collapsible-content hlist">

<ul><li><a href="Q-learning" title="Q-learning">Q-learning</a></li>

<li><a href="Policy_gradient_method" title="Policy gradient method">Policy gradient</a></li>

<li><a href="State%E2%80%93action%E2%80%93reward%E2%80%93state%E2%80%93action" title="State–action–reward–state–action">SARSA</a></li>

<li><a href="Temporal_difference_learning" title="Temporal difference learning">Temporal difference (TD)</a></li>

<li><a href="Multi-agent_reinforcement_learning" title="Multi-agent reinforcement learning">Multi-agent</a>

</tr><tr><td class="sidebar-content">

<ul><li><a href="Active_learning_(machine_learning)" title="Active learning (machine learning)">Active learning</a></li>

<li><a href="Crowdsourcing" title="Crowdsourcing">Crowdsourcing</a></li>

<li><a href="Human-in-the-loop" title="Human-in-the-loop">Human-in-the-loop</a></li>

<li><a href="Mechanistic_interpretability" title="Mechanistic interpretability">Mechanistic interpretability</a></li>

</tr><tr><td class="sidebar-content">

<ul><li><a href="Coefficient_of_determination" title="Coefficient of determination">Coefficient of determination</a></li>

<li><a href="Confusion_matrix" title="Confusion matrix">Confusion matrix</a></li>

<li><a href="Learning_curve_(machine_learning)" title="Learning curve (machine learning)">Learning curve</a></li>

<li><a href="Receiver_operating_characteristic" title="Receiver operating characteristic">ROC curve</a></li></ul></div></div></td>

</tr><tr><td class="sidebar-content">

<div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)">Mathematical foundations</div><div class="sidebar-list-content mw-collapsible-content hlist">

<ul><li><a href="Kernel_machines" class="mw-redirect" title="Kernel machines">Kernel machines</a></li>

<li><a href="Bias%E2%80%93variance_tradeoff" title="Bias–variance tradeoff">Bias–variance tradeoff</a></li>

<li><a href="Computational_learning_theory" title="Computational learning theory">Computational learning theory</a></li>

<li><a href="Empirical_risk_minimization" title="Empirical risk minimization">Empirical risk minimization</a></li>

<li><a href="Occam_learning" title="Occam learning">Occam learning</a></li>

<li><a href="Probably_approximately_correct_learning" title="Probably approximately correct learning">PAC learning</a></li>

<li><a href="Statistical_learning_theory" title="Statistical learning theory">Statistical learning</a></li>

<li><a href="Vapnik%E2%80%93Chervonenkis_theory" title="Vapnik–Chervonenkis theory">VC theory</a></li>

<li><a href="Topological_deep_learning" title="Topological deep learning">Topological deep learning</a></li></ul></div></div></td>

</tr><tr><td class="sidebar-content">

<div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)">Journals and conferences</div><div class="sidebar-list-content mw-collapsible-content hlist">

<li><a href="Conference_on_Neural_Information_Processing_Systems" title="Conference on Neural Information Processing Systems">NeurIPS</a></li>

<li><a href="International_Joint_Conference_on_Artificial_Intelligence" title="International Joint Conference on Artificial Intelligence">IJCAI</a></li>

</tr><tr><td class="sidebar-content">

<ul><li><a href="Glossary_of_artificial_intelligence" title="Glossary of artificial intelligence">Glossary of artificial intelligence</a></li>

<li><a href="List_of_datasets_for_machine-learning_research" title="List of datasets for machine-learning research">List of datasets for machine-learning research</a>

<ul><li><a href="List_of_datasets_in_computer_vision_and_image_processing" title="List of datasets in computer vision and image processing">List of datasets in computer vision and image processing</a></li></ul></li>

<li><a href="Outline_of_machine_learning" title="Outline of machine learning">Outline of machine learning</a></li></ul></div></div></td>

</tr><tr><td class="sidebar-navbar"><style data-mw-deduplicate="TemplateStyles:r1239400231">

/* start https://en.wikipedia.org/ */

.mw-parser-output .navbar{display:inline;font-size:88%;font-weight:normal}.mw-parser-output .navbar-collapse{float:left;text-align:left}.mw-parser-output .navbar-boxtext{word-spacing:0}.mw-parser-output .navbar ul{display:inline-block;white-space:nowrap;line-height:inherit}.mw-parser-output .navbar-brackets::before{margin-right:-0.125em;content:"[ "}.mw-parser-output .navbar-brackets::after{margin-left:-0.125em;content:" ]"}.mw-parser-output .navbar li{word-spacing:-0.125em}.mw-parser-output .navbar a>span,.mw-parser-output .navbar a>abbr{text-decoration:inherit}.mw-parser-output .navbar-mini abbr{font-variant:small-caps;border-bottom:none;text-decoration:none;cursor:inherit}.mw-parser-output .navbar-ct-full{font-size:114%;margin:0 7em}.mw-parser-output .navbar-ct-mini{font-size:114%;margin:0 4em}html.skin-theme-clientpref-night .mw-parser-output .navbar li a abbr{color:var(--color-base)!important}@media(prefers-color-scheme:dark){html.skin-theme-clientpref-os .mw-parser-output .navbar li a abbr{color:var(--color-base)!important}}@media print{.mw-parser-output .navbar{display:none!important}}

/* end https://en.wikipedia.org/ */

</style></td></tr></tbody></table>

<p>In <a href="Deep_learning" title="Deep learning">deep learning</a>, <b>weight initialization</b> or <b>parameter initialization</b> describes the initial step in creating a <a href="Neural_network_(machine_learning)" title="Neural network (machine learning)">neural network</a>. A neural network contains trainable parameters that are modified during training: weight initialization is the pre-training step of assigning initial values to these parameters.

</p><p>The choice of weight initialization method affects the speed of convergence, the scale of neural <a href="Activation_function" title="Activation function">activation</a> within the network, the scale of gradient signals during <a href="Backpropagation" title="Backpropagation">backpropagation</a>, and the quality of the final model. Proper initialization is necessary for avoiding issues such as <a href="Vanishing_gradient_problem" title="Vanishing gradient problem">vanishing and exploding gradients</a> and activation function <a href="Activation_function#Mathematical_details" title="Activation function">saturation</a>.

</p><p>Note that even though this article is titled "weight initialization", both weights and biases are used in a neural network as trainable parameters, so this article describes how both of these are initialized. Similarly, trainable parameters in <a href="Convolutional_neural_network" title="Convolutional neural network">convolutional neural networks</a> (CNNs) are called <a href="Kernel_(image_processing)" title="Kernel (image processing)">kernels</a> and biases, and this article also describes these.

</p>

<div class="mw-heading mw-heading2"><h2 id="Constant_initialization">Constant initialization</h2></div>

<p>We discuss the main methods of initialization in the context of a <a href="Multilayer_perceptron" title="Multilayer perceptron">multilayer perceptron</a> (MLP). Specific strategies for initializing other network architectures are discussed in later sections.

</p><p>For an MLP, there are only two kinds of trainable parameters, called weights and biases. Each layer <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle l}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle l}</annotation>

</semantics>

</math></span><img src="./829091f745070b9eb97a80244129025440a1cfac.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:0.693ex; height:2.176ex;" alt="{\displaystyle l}" loading="lazy"></span> contains a weight matrix <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle W^{(l)}\in \mathbb {R} ^{n_{l-1}\times n_{l}}}">

<msup>

</mrow>

</msup>

<msup>

</mrow>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mrow>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle W^{(l)}\in \mathbb {R} ^{n_{l-1}\times n_{l}}}</annotation>

</semantics>

</math></span><img src="./42bc3e87182360ce52b69873997c79e355c9ffa7.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:15.341ex; height:2.843ex;" alt="{\displaystyle W^{(l)}\in \mathbb {R} ^{n_{l-1}\times n_{l}}}" loading="lazy"></span>and a bias vector <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle b^{(l)}\in \mathbb {R} ^{n_{l}}}">

<msup>

</mrow>

</msup>

<msup>

</mrow>

<msub>

</mrow>

</msub>

</mrow>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle b^{(l)}\in \mathbb {R} ^{n_{l}}}</annotation>

</semantics>

</math></span><img src="./b02d53ca903b11c62da2f90fd6b5cb252a18a830.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:9.299ex; height:2.843ex;" alt="{\displaystyle b^{(l)}\in \mathbb {R} ^{n_{l}}}" loading="lazy"></span>, where <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle n_{l}}">

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle n_{l}}</annotation>

</semantics>

</math></span><img src="./99c0d834a6b3423aafc5780d141a1d4ccf853b18.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:2.117ex; height:2.009ex;" alt="{\displaystyle n_{l}}" loading="lazy"></span> is the number of neurons in that layer. A weight initialization method is an algorithm for setting the initial values for <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle W^{(l)},b^{(l)}}">

<msup>

</mrow>

</msup>

<msup>

</mrow>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle W^{(l)},b^{(l)}}</annotation>

</semantics>

</math></span><img src="./92485073de454571717306d3ada0276ceda6f7ae.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:8.543ex; height:3.176ex;" alt="{\displaystyle W^{(l)},b^{(l)}}" loading="lazy"></span> for each layer <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle l}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle l}</annotation>

</semantics>

</p><p>The simplest form is <b>zero initialization</b>:<span class="mwe-math-element mwe-math-element-block"><span class="mwe-math-mathml-display mwe-math-mathml-a11y" style="display: none;"><math display="block" xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle W^{(l)}=0,b^{(l)}=0}">

<msup>

</mrow>

</msup>

<msup>

</mrow>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle W^{(l)}=0,b^{(l)}=0}</annotation>

</semantics>

</math></span></span>Zero initialization is usually used for initializing biases, but it is not used for initializing weights, as it leads to <a href="Symmetry" title="Symmetry">symmetry</a> in the network, causing all neurons to learn the same features.

</p><p>In this page, we assume <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle b=0}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle b=0}</annotation>

</semantics>

</math></span><img src="./19206e7d4dab695ccb34c502eff0741e98dbdfc2.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:5.258ex; height:2.176ex;" alt="{\displaystyle b=0}" loading="lazy"></span> unless otherwise stated.

</p><p>Recurrent neural networks typically use activation functions with bounded range, such as sigmoid and tanh, since unbounded activation may cause exploding values. (Le, Jaitly, Hinton, 2015)<sup id="cite_ref-1" class="reference"><a href="#cite_note-1"><span class="cite-bracket">[</span>1<span class="cite-bracket">]</span></a></sup> suggested initializing weights in the recurrent parts of the network to identity and zero bias, similar to the idea of <a href="Residual_neural_network" title="Residual neural network">residual connections</a> and <a href="Long_short-term_memory" title="Long short-term memory">LSTM</a> with no forget gate.

</p><p>In most cases, the biases are initialized to zero, though some situations can use a nonzero initialization. For example, in multiplicative units, such as the forget gate of <a href="Long_short-term_memory" title="Long short-term memory">LSTM</a>, the bias can be initialized to 1 to allow good gradient signal through the gate.<sup id="cite_ref-2" class="reference"><a href="#cite_note-2"><span class="cite-bracket">[</span>2<span class="cite-bracket">]</span></a></sup> For neurons with <a href="Rectifier_(neural_networks)" title="Rectifier (neural networks)">ReLU</a> activation, one can initialize the bias to a small positive value like 0.1, so that the gradient is likely nonzero at initialization, avoiding the dying ReLU problem.<sup id="cite_ref-3" class="reference"><a href="#cite_note-3"><span class="cite-bracket">[</span>3<span class="cite-bracket">]</span></a></sup><sup class="reference nowrap"><span title="Page: 305">: 305 </span></sup><sup id="cite_ref-4" class="reference"><a href="#cite_note-4"><span class="cite-bracket">[</span>4<span class="cite-bracket">]</span></a></sup>

</p>

<div class="mw-heading mw-heading2"><h2 id="Random_initialization">Random initialization</h2></div>

<p><b>Random initialization</b> means sampling the weights from a <a href="Normal_distribution" title="Normal distribution">normal distribution</a> or a <a href="Continuous_uniform_distribution" title="Continuous uniform distribution">uniform distribution</a>, usually <a href="Independence_(probability_theory)" title="Independence (probability theory)">independently</a>.

</p>

<div class="mw-heading mw-heading3"><h3 id="LeCun_initialization">LeCun initialization</h3></div>

<p><b>LeCun initialization</b>, popularized in (LeCun et al., 1998),<sup id="cite_ref-:1_5-0" class="reference"><a href="#cite_note-:1-5"><span class="cite-bracket">[</span>5<span class="cite-bracket">]</span></a></sup> is designed to preserve the variance of neural activations during the forward pass.

</p><p>It samples each entry in <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle W^{(l)}}">

<msup>

</mrow>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle W^{(l)}}</annotation>

</semantics>

</math></span><img src="./559fe7596ddd43c18cbfc2cc93c60f345cdb257a.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:4.51ex; height:2.843ex;" alt="{\displaystyle W^{(l)}}" loading="lazy"></span> independently from a distribution with mean 0 and variance <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle 1/n_{l-1}}">

</mrow>

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle 1/n_{l-1}}</annotation>

</semantics>

</math></span><img src="./b5947bf3fdb251032e1fafa9bcaeeba084a920c4.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:6.543ex; height:2.843ex;" alt="{\displaystyle 1/n_{l-1}}" loading="lazy"></span>. For example, if the distribution is a <a href="Continuous_uniform_distribution" title="Continuous uniform distribution">continuous uniform distribution</a>, then the distribution is <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\mathcal {U}}(\pm {\sqrt {3/n_{l-1}}})}">

</mrow>

<msqrt>

</mrow>

<msub>

</mrow>

</msub>

</msqrt>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle {\mathcal {U}}(\pm {\sqrt {3/n_{l-1}}})}</annotation>

</semantics>

</math></span><img src="./c965935b024c6b7a146f49566da42a0aef39902e.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.838ex; margin-left: -0.038ex; width:14.119ex; height:4.843ex;" alt="{\displaystyle {\mathcal {U}}(\pm {\sqrt {3/n_{l-1}}})}" loading="lazy"></span>.

</p>

<div class="mw-heading mw-heading3"><h3 id="Glorot_initialization">Glorot initialization</h3></div>

<p><b>Glorot initialization</b> (or <b>Xavier initialization</b>) was proposed by Xavier Glorot and <a href="Yoshua_Bengio" title="Yoshua Bengio">Yoshua Bengio</a>.<sup id="cite_ref-6" class="reference"><a href="#cite_note-6"><span class="cite-bracket">[</span>6<span class="cite-bracket">]</span></a></sup> It was designed as a compromise between two goals: to preserve activation variance during the forward pass and to preserve gradient variance during the backward pass.

</p><p>For uniform initialization, it samples each entry in <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle W^{(l)}}">

<msup>

</mrow>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle W^{(l)}}</annotation>

</semantics>

</math></span><img src="./559fe7596ddd43c18cbfc2cc93c60f345cdb257a.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:4.51ex; height:2.843ex;" alt="{\displaystyle W^{(l)}}" loading="lazy"></span> <a href="Independent_and_identically_distributed_random_variables" title="Independent and identically distributed random variables">independently and identically</a> from <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\mathcal {U}}(\pm {\sqrt {6/(n_{l+1}+n_{l-1})}})}">

</mrow>

<msqrt>

</mrow>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</msqrt>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle {\mathcal {U}}(\pm {\sqrt {6/(n_{l+1}+n_{l-1})}})}</annotation>

</semantics>

</math></span><img src="./6cfbbb6540b746a7d18b7143778d3ed5e996fef6.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.838ex; margin-left: -0.038ex; width:22.986ex; height:4.843ex;" alt="{\displaystyle {\mathcal {U}}(\pm {\sqrt {6/(n_{l+1}+n_{l-1})}})}" loading="lazy"></span>. In the context, <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle n_{l-1}}">

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle n_{l-1}}</annotation>

</semantics>

</math></span><img src="./50a620147c6d183a3450cee16e157cb17de84bb1.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:4.218ex; height:2.009ex;" alt="{\displaystyle n_{l-1}}" loading="lazy"></span> is also called the "<b>fan-in</b>", and <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle n_{l+1}}">

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle n_{l+1}}</annotation>

</semantics>

</math></span><img src="./0fe9b11ca071a5fb27f3a394713068ecc1ab9ea7.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:4.218ex; height:2.009ex;" alt="{\displaystyle n_{l+1}}" loading="lazy"></span> the "<b>fan-out</b>". When the fan-in and fan-out are equal, then Glorot initialization is the same as LeCun initialization.

</p>

<div class="mw-heading mw-heading3"><h3 id="He_initialization">He initialization</h3></div>

<p>As Glorot initialization performs poorly for ReLU activation,<sup id="cite_ref-7" class="reference"><a href="#cite_note-7"><span class="cite-bracket">[</span>7<span class="cite-bracket">]</span></a></sup> <b>He initialization</b> (or <b>Kaiming initialization</b>) was proposed by <a href="Kaiming_He" title="Kaiming He">Kaiming He</a> et al.<sup id="cite_ref-8" class="reference"><a href="#cite_note-8"><span class="cite-bracket">[</span>8<span class="cite-bracket">]</span></a></sup> for networks with <a href="ReLU" class="mw-redirect" title="ReLU">ReLU</a> activation. It samples each entry in <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle W^{(l)}}">

<msup>

</mrow>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle W^{(l)}}</annotation>

</semantics>

</math></span><img src="./559fe7596ddd43c18cbfc2cc93c60f345cdb257a.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:4.51ex; height:2.843ex;" alt="{\displaystyle W^{(l)}}" loading="lazy"></span> from <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\mathcal {N}}(0,{\sqrt {2/n_{l-1}}})}">

</mrow>

<msqrt>

</mrow>

<msub>

</mrow>

</msub>

</msqrt>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle {\mathcal {N}}(0,{\sqrt {2/n_{l-1}}})}</annotation>

</semantics>

</math></span><img src="./04d3d9543c34fc0db9ee5f5ffebc47233c414762.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.838ex; margin-left: -0.062ex; width:15.209ex; height:4.843ex;" alt="{\displaystyle {\mathcal {N}}(0,{\sqrt {2/n_{l-1}}})}" loading="lazy"></span>.

</p>

<div class="mw-heading mw-heading3"><h3 id="Orthogonal_initialization">Orthogonal initialization</h3></div>

<p>(Saxe et al. 2013)<sup id="cite_ref-9" class="reference"><a href="#cite_note-9"><span class="cite-bracket">[</span>9<span class="cite-bracket">]</span></a></sup> proposed <b>orthogonal initialization</b>: initializing weight matrices as uniformly random (according to the <a href="Haar_measure" title="Haar measure">Haar measure</a>) <a href="Semi-orthogonal_matrix" title="Semi-orthogonal matrix">semi-orthogonal matrices</a>, multiplied by a factor that depends on the activation function of the layer. It was designed so that if one initializes a deep linear network this way, then its training time until convergence is independent of depth.<sup id="cite_ref-10" class="reference"><a href="#cite_note-10"><span class="cite-bracket">[</span>10<span class="cite-bracket">]</span></a></sup>

</p><p>Sampling a uniformly random semi-orthogonal matrix can be done by initializing <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle X}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle X}</annotation>

</semantics>

</math></span><img src="./68baa052181f707c662844a465bfeeb135e82bab.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.98ex; height:2.176ex;" alt="{\displaystyle X}" loading="lazy"></span> by IID sampling its entries from a standard normal distribution, then calculate <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \left(XX^{\top }\right)^{-1/2}X}">

<msup>

<mrow>

<mrow>

<msup>

</mrow>

</msup>

</mrow>

</mrow>

</mrow>

</mrow>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \left(XX^{\top }\right)^{-1/2}X}</annotation>

</semantics>

</math></span><img src="./edc64c49a6338892f3474b2f33e9eb2b5c667b32.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; width:13.574ex; height:3.843ex;" alt="{\displaystyle \left(XX^{\top }\right)^{-1/2}X}" loading="lazy"></span> or its transpose, depending on whether <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle X}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle X}</annotation>

</semantics>

</math></span><img src="./68baa052181f707c662844a465bfeeb135e82bab.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.98ex; height:2.176ex;" alt="{\displaystyle X}" loading="lazy"></span> is tall or wide.<sup id="cite_ref-:2_11-0" class="reference"><a href="#cite_note-:2-11"><span class="cite-bracket">[</span>11<span class="cite-bracket">]</span></a></sup>

</p><p>For CNN kernels with odd widths and heights, orthogonal initialization is done this way: initialize the central point by a semi-orthogonal matrix, and fill the other entries with zero. As an illustration, a kernel <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle K}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle K}</annotation>

</semantics>

</math></span><img src="./2b76fce82a62ed5461908f0dc8f037de4e3686b0.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:2.066ex; height:2.176ex;" alt="{\displaystyle K}" loading="lazy"></span> of shape <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle 3\times 3\times c\times c'}">

<msup>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle 3\times 3\times c\times c'}</annotation>

</semantics>

</math></span><img src="./aca9bd37e541c34863a671ffd7e926aeeebb5fd6.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:13.545ex; height:2.509ex;" alt="{\displaystyle 3\times 3\times c\times c'}" loading="lazy"></span> is initialized by filling <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle K[2,2,:,:]}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle K[2,2,:,:]}</annotation>

</semantics>

</math></span><img src="./bd96c9c0b6df2e5507e91d8a6744338f4a8c6a83.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:10.08ex; height:2.843ex;" alt="{\displaystyle K[2,2,:,:]}" loading="lazy"></span> with the entries of a random semi-orthogonal matrix of shape <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle c\times c'}">

<msup>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle c\times c'}</annotation>

</semantics>

</math></span><img src="./ac25f3b0f82289e639623a0c0df9db08f7c3cbb5.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:5.539ex; height:2.509ex;" alt="{\displaystyle c\times c'}" loading="lazy"></span>, and the other entries with zero. (Balduzzi et al., 2017)<sup id="cite_ref-12" class="reference"><a href="#cite_note-12"><span class="cite-bracket">[</span>12<span class="cite-bracket">]</span></a></sup> used it with stride 1 and zero-padding. This is sometimes called the <b>Orthogonal Delta initialization</b>.<sup id="cite_ref-:2_11-1" class="reference"><a href="#cite_note-:2-11"><span class="cite-bracket">[</span>11<span class="cite-bracket">]</span></a></sup><sup id="cite_ref-13" class="reference"><a href="#cite_note-13"><span class="cite-bracket">[</span>13<span class="cite-bracket">]</span></a></sup>

</p><p>Related to this approach, <b>unitary initialization</b> proposes to parameterize the weight matrices to be <a href="Unitary_matrix" title="Unitary matrix">unitary matrices</a>, with the result that at initialization they are random unitary matrices (and throughout training, they remain unitary). This is found to improve long-sequence modelling in LSTM.<sup id="cite_ref-14" class="reference"><a href="#cite_note-14"><span class="cite-bracket">[</span>14<span class="cite-bracket">]</span></a></sup><sup id="cite_ref-15" class="reference"><a href="#cite_note-15"><span class="cite-bracket">[</span>15<span class="cite-bracket">]</span></a></sup>

</p><p>Orthogonal initialization has been generalized to <b>layer-sequential unit-variance (LSUV) initialization</b>. It is a data-dependent initialization method, and can be used in <a href="Convolutional_neural_network" title="Convolutional neural network">convolutional neural networks</a>. It first initializes weights of each convolution or fully connected layer with orthonormal matrices. Then, proceeding from the first to the last layer, it runs a forward pass on a random minibatch, and divides the layer's weights by the standard deviation of its output, so that its output has variance approximately 1.<sup id="cite_ref-16" class="reference"><a href="#cite_note-16"><span class="cite-bracket">[</span>16<span class="cite-bracket">]</span></a></sup><sup id="cite_ref-17" class="reference"><a href="#cite_note-17"><span class="cite-bracket">[</span>17<span class="cite-bracket">]</span></a></sup>

</p>

<div class="mw-heading mw-heading3"><h3 id="Fixup_initialization">Fixup initialization</h3></div>

<p>In 2015, the introduction of <a href="Residual_neural_network" title="Residual neural network">residual connections</a> allowed very deep neural networks to be trained, much deeper than the ~20 layers of the previous state of the art (such as the <a href="VGGNet" title="VGGNet">VGG-19</a>). Residual connections gave rise to their own weight initialization problems and strategies. These are sometimes called "normalization-free" methods, since using residual connection could stabilize the training of a deep neural network so much that <a href="Normalization_(machine_learning)" title="Normalization (machine learning)">normalizations</a> become unnecessary.

</p><p><b>Fixup initialization</b> is designed specifically for networks with <a href="Residual_neural_network" title="Residual neural network">residual connections</a> and without batch normalization, as follows:<sup id="cite_ref-18" class="reference"><a href="#cite_note-18"><span class="cite-bracket">[</span>18<span class="cite-bracket">]</span></a></sup>

</p>

<ol><li>Initialize the classification layer and the last layer of each residual branch to 0.</li>

<li>Initialize every other layer using a standard method (such as He initialization), and scale only the weight layers inside residual branches by <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle L^{-{\frac {1}{2m-2}}}}">

<msup>

<mfrac>

<mrow>

</mrow>

</mfrac>

</mrow>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle L^{-{\frac {1}{2m-2}}}}</annotation>

</semantics>

</math></span><img src="./5406cd98dfbc43457535c0523d389a793cf5f848.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:7.474ex; height:3.676ex;" alt="{\displaystyle L^{-{\frac {1}{2m-2}}}}" loading="lazy"></span>.</li>

<li>Add a scalar multiplier (initialized at 1) in every branch and a scalar bias (initialized at 0) before each convolution, linear, and element-wise activation layer.</li></ol>

<p>Similarly, <b>T-Fixup initialization</b> is designed for <a href="Transformer_(deep_learning_architecture)" title="Transformer (deep learning architecture)">Transformers</a> without <a href="Layer_normalization" class="mw-redirect" title="Layer normalization">layer normalization</a>.<sup id="cite_ref-19" class="reference"><a href="#cite_note-19"><span class="cite-bracket">[</span>19<span class="cite-bracket">]</span></a></sup><sup class="reference nowrap"><span title="Page: 9">: 9 </span></sup>

</p>

<div class="mw-heading mw-heading3"><h3 id="Others">Others</h3></div>

<p>Instead of initializing all weights with random values on the order of <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle O(1/{\sqrt {n}})}">

</mrow>

<msqrt>

</msqrt>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle O(1/{\sqrt {n}})}</annotation>

</semantics>

</math></span><img src="./f61ed26c4b4b0c3c3f29bf4d3c6128bfe26510f9.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; width:9.238ex; height:3.009ex;" alt="{\displaystyle O(1/{\sqrt {n}})}" loading="lazy"></span>, <b><a href="Sparse_matrix" title="Sparse matrix">sparse</a> initialization</b> initialized only a small subset of the weights with larger random values, and the other weights zero, so that the total variance is still on the order of <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle O(1)}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle O(1)}</annotation>

</semantics>

</math></span><img src="./e66384bc40452c5452f33563fe0e27e803b0cc21.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:4.745ex; height:2.843ex;" alt="{\displaystyle O(1)}" loading="lazy"></span>.<sup id="cite_ref-:0_20-0" class="reference"><a href="#cite_note-:0-20"><span class="cite-bracket">[</span>20<span class="cite-bracket">]</span></a></sup>

</p><p><b>Random walk initialization</b> was designed for MLP so that during backpropagation, the L2 norm of gradient at each layer performs an unbiased random walk as one moves from the last layer to the first.<sup id="cite_ref-21" class="reference"><a href="#cite_note-21"><span class="cite-bracket">[</span>21<span class="cite-bracket">]</span></a></sup>

</p><p><b>Looks linear initialization</b> was designed to allow the neural network to behave like a deep linear network at initialization, since <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle W\;\mathrm {ReLU} (x)-W\;\mathrm {ReLU} (-x)=Wx}">

</mrow>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle W\;\mathrm {ReLU} (x)-W\;\mathrm {ReLU} (-x)=Wx}</annotation>

</semantics>

</math></span><img src="./c2a014e31339f59f526838e018b490dad4c3c92b.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:35.828ex; height:2.843ex;" alt="{\displaystyle W\;\mathrm {ReLU} (x)-W\;\mathrm {ReLU} (-x)=Wx}" loading="lazy"></span>. It initializes a matrix <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle W}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle W}</annotation>

</semantics>

</math></span><img src="./54a9c4c547f4d6111f81946cad242b18298d70b7.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:2.435ex; height:2.176ex;" alt="{\displaystyle W}" loading="lazy"></span> of shape <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \mathbb {R} ^{{\frac {n}{2}}\times m}}">

<msup>

</mrow>

<mfrac>

</mfrac>

</mrow>

</mrow>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \mathbb {R} ^{{\frac {n}{2}}\times m}}</annotation>

</semantics>

</math></span><img src="./1d2794cd3dc417ae2e590ad9a0a68ec814e79612.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:6.268ex; height:3.176ex;" alt="{\displaystyle \mathbb {R} ^{{\frac {n}{2}}\times m}}" loading="lazy"></span> by any method, such as orthogonal initialization, then let the <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \mathbb {R} ^{n\times m}}">

<msup>

</mrow>

</mrow>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \mathbb {R} ^{n\times m}}</annotation>

</semantics>

</math></span><img src="./94cb9d4a4bc62be08e90334fa87531883301210b.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:5.618ex; height:2.343ex;" alt="{\displaystyle \mathbb {R} ^{n\times m}}" loading="lazy"></span> weight matrix to be the concatenation of <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle W,-W}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle W,-W}</annotation>

</semantics>

</math></span><img src="./34b1854ce295f98ef055f1e1ec087051bb8fde5d.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:7.713ex; height:2.509ex;" alt="{\displaystyle W,-W}" loading="lazy"></span>.<sup id="cite_ref-22" class="reference"><a href="#cite_note-22"><span class="cite-bracket">[</span>22<span class="cite-bracket">]</span></a></sup>

</p>

<div class="mw-heading mw-heading2"><h2 id="Miscellaneous">Miscellaneous</h2></div>

<p>For <a href="Hyperbolic_functions" title="Hyperbolic functions">hyperbolic tangent</a> activation function, a particular scaling is sometimes used: <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle 1.7159\tanh(2x/3)}">

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle 1.7159\tanh(2x/3)}</annotation>

</semantics>

</math></span><img src="./cff7b52bb1e019b3a73c2d2b399144011ebc69cd.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:18.125ex; height:2.843ex;" alt="{\displaystyle 1.7159\tanh(2x/3)}" loading="lazy"></span>. This was sometimes called "<b><a href="Yann_LeCun" title="Yann LeCun">LeCun</a>'s tanh</b>". It was designed so that it maps the interval <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle [-1,+1]}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle [-1,+1]}</annotation>

</semantics>

</math></span><img src="./daa72f1a806823ec94fda7922597b19cbda684f4.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:8.269ex; height:2.843ex;" alt="{\displaystyle [-1,+1]}" loading="lazy"></span> to itself, thus ensuring that the overall gain is around 1 in "normal operating conditions", and that <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle |f''(x)|}">

</mrow>

<msup>

</msup>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle |f''(x)|}</annotation>

</semantics>

</math></span><img src="./b61af5dbf0875647daae315da25eab0296039147.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:6.89ex; height:3.009ex;" alt="{\displaystyle |f''(x)|}" loading="lazy"></span> is at maximum when <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle x=-1,+1}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle x=-1,+1}</annotation>

</semantics>

</math></span><img src="./d88ae9e813231f3a6e0e0e1c6a579c44fe7b040c.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:11.403ex; height:2.509ex;" alt="{\displaystyle x=-1,+1}" loading="lazy"></span>, which improves convergence at the end of training.<sup id="cite_ref-23" class="reference"><a href="#cite_note-23"><span class="cite-bracket">[</span>23<span class="cite-bracket">]</span></a></sup><sup id="cite_ref-:1_5-1" class="reference"><a href="#cite_note-:1-5"><span class="cite-bracket">[</span>5<span class="cite-bracket">]</span></a></sup>

</p><p>In <b>self-normalizing neural networks</b>, the <a href="Activation_function" title="Activation function">SELU activation function</a> <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \mathrm {SELU} (x)=\lambda {\begin{cases}x&{\text{if }}x>0\\\alpha e^{x}-\alpha &{\text{if }}x\leq 0\end{cases}}}">

</mrow>

<mrow>

<mtr>

<mtd>

</mtd>

<mtd>

</mrow>

</mtd>

</mtr>

<mtr>

<mtd>

<msup>

</mrow>

</msup>

</mtd>

<mtd>

</mrow>

</mtd>

</mtr>

</mtable>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \mathrm {SELU} (x)=\lambda {\begin{cases}x&{\text{if }}x>0\\\alpha e^{x}-\alpha &{\text{if }}x\leq 0\end{cases}}}</annotation>

</semantics>

</math></span><img src="./9a9e127885f5211bb99e309340402ab09968c63d.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -2.505ex; width:34.083ex; height:6.176ex;" alt="{\displaystyle \mathrm {SELU} (x)=\lambda {\begin{cases}x&{\text{if }}x>0\\\alpha e^{x}-\alpha &{\text{if }}x\leq 0\end{cases}}}" loading="lazy"></span> with parameters <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \lambda \approx 1.0507,\alpha \approx 1.6733}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \lambda \approx 1.0507,\alpha \approx 1.6733}</annotation>

</semantics>

</math></span><img src="./7b51afc14b5ffaaf52ac52c06a13f7531eaa321c.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:22.992ex; height:2.509ex;" alt="{\displaystyle \lambda \approx 1.0507,\alpha \approx 1.6733}" loading="lazy"></span> makes it such that the mean and variance of the output of each layer has <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle (0,1)}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle (0,1)}</annotation>

</semantics>

</math></span><img src="./c79c6838e423c1ed3c7ea532a56dc9f9dae8290b.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:5.168ex; height:2.843ex;" alt="{\displaystyle (0,1)}" loading="lazy"></span> as an attracting fixed-point. This makes initialization less important, though they recommend initializing weights randomly with variance <span class="mwe-math-element mwe-math-element-inline"><span class="mwe-math-mathml-inline mwe-math-mathml-a11y" style="display: none;"><math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle 1/n_{l-1}}">

</mrow>

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle 1/n_{l-1}}</annotation>

</semantics>

</p>

<div class="mw-heading mw-heading2"><h2 id="History">History</h2></div>

<p>Random weight initialization was used since <a href="Frank_Rosenblatt" title="Frank Rosenblatt">Frank Rosenblatt</a>'s <a href="Perceptron" title="Perceptron">perceptrons</a>. An early work that described weight initialization specifically was (LeCun et al., 1998).<sup id="cite_ref-:1_5-2" class="reference"><a href="#cite_note-:1-5"><span class="cite-bracket">[</span>5<span class="cite-bracket">]</span></a></sup>

</p><p>Before the 2010s era of deep learning, it was common to initialize models by "generative pre-training" using an unsupervised learning algorithm that is not backpropagation, as it was difficult to directly train deep neural networks by backpropagation.<sup id="cite_ref-25" class="reference"><a href="#cite_note-25"><span class="cite-bracket">[</span>25<span class="cite-bracket">]</span></a></sup><sup id="cite_ref-26" class="reference"><a href="#cite_note-26"><span class="cite-bracket">[</span>26<span class="cite-bracket">]</span></a></sup> For example, a <a href="Deep_belief_network" title="Deep belief network">deep belief network</a> was trained by using <a href="Contrastive_divergence" class="mw-redirect" title="Contrastive divergence">contrastive divergence</a> layer by layer, starting from the bottom.<sup id="cite_ref-27" class="reference"><a href="#cite_note-27"><span class="cite-bracket">[</span>27<span class="cite-bracket">]</span></a></sup>

</p><p>(Martens, 2010)<sup id="cite_ref-:0_20-1" class="reference"><a href="#cite_note-:0-20"><span class="cite-bracket">[</span>20<span class="cite-bracket">]</span></a></sup> proposed Hessian-free Optimization, a <a href="Quasi-Newton_method" title="Quasi-Newton method">quasi-Newton method</a> to directly train deep networks. The work generated considerable excitement that initializing networks without pre-training phase was possible.<sup id="cite_ref-28" class="reference"><a href="#cite_note-28"><span class="cite-bracket">[</span>28<span class="cite-bracket">]</span></a></sup> However, a 2013 paper demonstrated that with well-chosen hyperparameters, <a href="Momentum_gradient_descent" class="mw-redirect" title="Momentum gradient descent">momentum gradient descent</a> with weight initialization was sufficient for training neural networks, without needing either quasi-Newton method or generative pre-training, a combination that is still in use as of 2024.<sup id="cite_ref-29" class="reference"><a href="#cite_note-29"><span class="cite-bracket">[</span>29<span class="cite-bracket">]</span></a></sup>

</p><p>Since then, the impact of initialization on tuning the variance has become less important, with methods developed to automatically tune variance, like <a href="Batch_normalization" title="Batch normalization">batch normalization</a> tuning the variance of the forward pass,<sup id="cite_ref-30" class="reference"><a href="#cite_note-30"><span class="cite-bracket">[</span>30<span class="cite-bracket">]</span></a></sup> and <a href="Adam_optimizer" class="mw-redirect" title="Adam optimizer">momentum-based optimizers</a> tuning the variance of the backward pass.<sup id="cite_ref-31" class="reference"><a href="#cite_note-31"><span class="cite-bracket">[</span>31<span class="cite-bracket">]</span></a></sup>

</p><p>There is a tension between using careful weight initialization to decrease the need for normalization, and using normalization to decrease the need for careful weight initialization, with each approach having its tradeoffs. For example, batch normalization causes training examples in the minibatch to become dependent, an undesirable trait, while weight initialization is architecture-dependent.<sup id="cite_ref-32" class="reference"><a href="#cite_note-32"><span class="cite-bracket">[</span>32<span class="cite-bracket">]</span></a></sup>

</p>

<ul><li><a href="Backpropagation" title="Backpropagation">Backpropagation</a></li>

<li><a href="Normalization_(machine_learning)" title="Normalization (machine learning)">Normalization (machine learning)</a></li>

<li><a href="Gradient_descent" title="Gradient descent">Gradient descent</a></li>

<li><a href="Vanishing_gradient_problem" title="Vanishing gradient problem">Vanishing gradient problem</a></li></ul>

<div class="mw-heading mw-heading2"><h2 id="References">References</h2></div>

/* start https://en.wikipedia.org/ */

.mw-parser-output .reflist{margin-bottom:0.5em;list-style-type:decimal}@media screen{.mw-parser-output .reflist{font-size:90%}}.mw-parser-output .reflist .references{font-size:100%;margin-bottom:0;list-style-type:inherit}.mw-parser-output .reflist-columns-2{column-width:30em}.mw-parser-output .reflist-columns-3{column-width:25em}.mw-parser-output .reflist-columns{margin-top:0.3em}.mw-parser-output .reflist-columns ol{margin-top:0}.mw-parser-output .reflist-columns li{page-break-inside:avoid;break-inside:avoid-column}.mw-parser-output .reflist-upper-alpha{list-style-type:upper-alpha}.mw-parser-output .reflist-upper-roman{list-style-type:upper-roman}.mw-parser-output .reflist-lower-alpha{list-style-type:lower-alpha}.mw-parser-output .reflist-lower-greek{list-style-type:lower-greek}.mw-parser-output .reflist-lower-roman{list-style-type:lower-roman}

/* end https://en.wikipedia.org/ */

</style><div class="reflist reflist-columns references-column-width" style="column-width: 30em;">

/* start https://en.wikipedia.org/ */

.mw-parser-output cite.citation{font-style:inherit;word-wrap:break-word}.mw-parser-output .citation q{quotes:"\"""\"""'""'"}.mw-parser-output .citation:target{background-color:rgba(0,127,255,0.133)}.mw-parser-output .id-lock-free.id-lock-free a{background:url("./mw/Lock-green.svg")right 0.1em center/9px no-repeat}.mw-parser-output .id-lock-limited.id-lock-limited a,.mw-parser-output .id-lock-registration.id-lock-registration a{background:url("./mw/Lock-gray-alt-2.svg")right 0.1em center/9px no-repeat}.mw-parser-output .id-lock-subscription.id-lock-subscription a{background:url("./mw/Lock-red-alt-2.svg")right 0.1em center/9px no-repeat}.mw-parser-output .cs1-ws-icon a{background:url("./mw/Wikisource-logo.svg")right 0.1em center/12px no-repeat}body:not(.skin-timeless):not(.skin-minerva) .mw-parser-output .id-lock-free a,body:not(.skin-timeless):not(.skin-minerva) .mw-parser-output .id-lock-limited a,body:not(.skin-timeless):not(.skin-minerva) .mw-parser-output .id-lock-registration a,body:not(.skin-timeless):not(.skin-minerva) .mw-parser-output .id-lock-subscription a,body:not(.skin-timeless):not(.skin-minerva) .mw-parser-output .cs1-ws-icon a{background-size:contain;padding:0 1em 0 0}.mw-parser-output .cs1-code{color:inherit;background:inherit;border:none;padding:inherit}.mw-parser-output .cs1-hidden-error{display:none;color:var(--color-error,#d33)}.mw-parser-output .cs1-visible-error{color:var(--color-error,#d33)}.mw-parser-output .cs1-maint{display:none;color:#085;margin-left:0.3em}.mw-parser-output .cs1-kern-left{padding-left:0.2em}.mw-parser-output .cs1-kern-right{padding-right:0.2em}.mw-parser-output .citation .mw-selflink{font-weight:inherit}@media screen{.mw-parser-output .cs1-format{font-size:95%}html.skin-theme-clientpref-night .mw-parser-output .cs1-maint{color:#18911f}}@media screen and (prefers-color-scheme:dark){html.skin-theme-clientpref-os .mw-parser-output .cs1-maint{color:#18911f}}

/* end https://en.wikipedia.org/ */

</style><cite id="CITEREFLeJaitlyHinton2015" class="citation arxiv cs1">Le, Quoc V.; Jaitly, Navdeep; Hinton, Geoffrey E. (2015). "A Simple Way to Initialize Recurrent Networks of Rectified Linear Units". <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://arxiv.org/abs/1504.00941">1504.00941</a></span> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/cs.NE">cs.NE</a>].</cite></span>

</li>

<li id="cite_note-2"><span class="mw-cite-backlink"><b><a href="#cite_ref-2">^</a></b></span> <span class="reference-text"><cite id="CITEREFJozefowiczZarembaSutskever2015" class="citation journal cs1">Jozefowicz, Rafal; Zaremba, Wojciech; Sutskever, Ilya (2015-06-01). <a rel="nofollow" class="external text" href="https://proceedings.mlr.press/v37/jozefowicz15.html">"An Empirical Exploration of Recurrent Network Architectures"</a>. <i>Proceedings of the 32nd International Conference on Machine Learning</i>. PMLR: <span class="nowrap">2342–</span>2350.</cite></span>

</li>

<li id="cite_note-3"><span class="mw-cite-backlink"><b><a href="#cite_ref-3">^</a></b></span> <span class="reference-text"><cite id="CITEREFGoodfellowBengioCourville2016" class="citation book cs1">Goodfellow, Ian; Bengio, Yoshua; Courville, Aaron (2016). <i>Deep learning</i>. Adaptive computation and machine learning. Cambridge, Massachusetts: The MIT Press. <a href="ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <bdi>978-0-262-03561-3</bdi>.</cite></span>

</li>

<li id="cite_note-4"><span class="mw-cite-backlink"><b><a href="#cite_ref-4">^</a></b></span> <span class="reference-text"><cite id="CITEREFLuShinSuKarniadakis2019" class="citation journal cs1">Lu, Lu; Shin, Yeonjong; Su, Yanhui; Karniadakis, George Em (2019). "Dying ReLU and Initialization: Theory and Numerical Examples". <i>Communications in Computational Physics</i>. <b>28</b> (5): <span class="nowrap">1671–</span>1706. <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://arxiv.org/abs/1903.06733">1903.06733</a></span>. <a href="Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.4208%2Fcicp.OA-2020-0165">10.4208/cicp.OA-2020-0165</a>.</cite></span>

</li>

<li id="cite_note-:1-5"><span class="mw-cite-backlink">^ <a href="#cite_ref-:1_5-0"><sup><i><b>a</b></i></sup></a> <a href="#cite_ref-:1_5-1"><sup><i><b>b</b></i></sup></a> <a href="#cite_ref-:1_5-2"><sup><i><b>c</b></i></sup></a></span> <span class="reference-text"><cite id="CITEREFLeCunBottouOrrMüller1998" class="citation cs2">LeCun, Yann; Bottou, Leon; Orr, Genevieve B.; Müller, Klaus -Robert (1998), Orr, Genevieve B.; Müller, Klaus-Robert (eds.), <span class="id-lock-subscription" title="Paid subscription required"><a rel="nofollow" class="external text" href="https://link.springer.com/chapter/10.1007/3-540-49430-8_2">"Efficient BackProp"</a></span>, <i>Neural Networks: Tricks of the Trade</i>, Berlin, Heidelberg: Springer, pp. <span class="nowrap">9–</span>50, <a href="Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1007%2F3-540-49430-8_2">10.1007/3-540-49430-8_2</a>, <a href="ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <bdi>978-3-540-49430-0</bdi><span class="reference-accessdate">, retrieved <span class="nowrap">2024-10-05</span></span></cite></span>

</li>

<li id="cite_note-6"><span class="mw-cite-backlink"><b><a href="#cite_ref-6">^</a></b></span> <span class="reference-text"><cite id="CITEREFGlorotBengio2010" class="citation journal cs1">Glorot, Xavier; Bengio, Yoshua (2010-03-31). <a rel="nofollow" class="external text" href="https://proceedings.mlr.press/v9/glorot10a">"Understanding the difficulty of training deep feedforward neural networks"</a>. <i>Proceedings of the Thirteenth International Conference on Artificial Intelligence and Statistics</i>. JMLR Workshop and Conference Proceedings: <span class="nowrap">249–</span>256.</cite></span>

</li>

<li id="cite_note-7"><span class="mw-cite-backlink"><b><a href="#cite_ref-7">^</a></b></span> <span class="reference-text"><cite id="CITEREFKumar2017" class="citation arxiv cs1">Kumar, Siddharth Krishna (2017). "On weight initialization in deep neural networks". <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://arxiv.org/abs/1704.08863">1704.08863</a></span> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/cs.LG">cs.LG</a>].</cite></span>

</li>

<li id="cite_note-8"><span class="mw-cite-backlink"><b><a href="#cite_ref-8">^</a></b></span> <span class="reference-text"><cite id="CITEREFHeZhangRenSun2015" class="citation arxiv cs1">He, Kaiming; Zhang, Xiangyu; Ren, Shaoqing; Sun, Jian (2015). "Delving Deep into Rectifiers: Surpassing Human-Level Performance on ImageNet Classification". <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://arxiv.org/abs/1502.01852">1502.01852</a></span> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/cs.CV">cs.CV</a>].</cite></span>

</li>

<li id="cite_note-9"><span class="mw-cite-backlink"><b><a href="#cite_ref-9">^</a></b></span> <span class="reference-text"><cite id="CITEREFSaxeMcClellandGanguli2013" class="citation arxiv cs1">Saxe, Andrew M.; McClelland, James L.; Ganguli, Surya (2013). "Exact solutions to the nonlinear dynamics of learning in deep linear neural networks". <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://arxiv.org/abs/1312.6120">1312.6120</a></span> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/cs.NE">cs.NE</a>].</cite></span>

</li>

<li id="cite_note-10"><span class="mw-cite-backlink"><b><a href="#cite_ref-10">^</a></b></span> <span class="reference-text"><cite id="CITEREFHuXiaoPennington2020" class="citation arxiv cs1">Hu, Wei; Xiao, Lechao; Pennington, Jeffrey (2020). "Provable Benefit of Orthogonal Initialization in Optimizing Deep Linear Networks". <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://arxiv.org/abs/2001.05992">2001.05992</a></span> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/cs.LG">cs.LG</a>].</cite></span>

</li>

<li id="cite_note-:2-11"><span class="mw-cite-backlink">^ <a href="#cite_ref-:2_11-0"><sup><i><b>a</b></i></sup></a> <a href="#cite_ref-:2_11-1"><sup><i><b>b</b></i></sup></a></span> <span class="reference-text"><cite id="CITEREFMartensBallardDesjardinsSwirszcz2021" class="citation arxiv cs1">Martens, James; Ballard, Andy; Desjardins, Guillaume; Swirszcz, Grzegorz; Dalibard, Valentin; Sohl-Dickstein, Jascha; Schoenholz, Samuel S. (2021). "Rapid training of deep neural networks without skip connections or normalization layers using Deep Kernel Shaping". <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://arxiv.org/abs/2110.01765">2110.01765</a></span> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/cs.LG">cs.LG</a>].</cite></span>

</li>

<li id="cite_note-12"><span class="mw-cite-backlink"><b><a href="#cite_ref-12">^</a></b></span> <span class="reference-text"><cite id="CITEREFBalduzziFreanLearyLewis2017" class="citation journal cs1">Balduzzi, David; Frean, Marcus; Leary, Lennox; Lewis, J. P.; Ma, Kurt Wan-Duo; McWilliams, Brian (2017-07-17). <a rel="nofollow" class="external text" href="https://proceedings.mlr.press/v70/balduzzi17b.html">"The Shattered Gradients Problem: If resnets are the answer, then what is the question?"</a>. <i>Proceedings of the 34th International Conference on Machine Learning</i>. PMLR: <span class="nowrap">342–</span>350.</cite></span>

</li>

<li id="cite_note-13"><span class="mw-cite-backlink"><b><a href="#cite_ref-13">^</a></b></span> <span class="reference-text"><cite id="CITEREFXiaoBahriSohl-DicksteinSchoenholz2018" class="citation journal cs1">Xiao, Lechao; Bahri, Yasaman; Sohl-Dickstein, Jascha; Schoenholz, Samuel; Pennington, Jeffrey (2018-07-03). <a rel="nofollow" class="external text" href="https://proceedings.mlr.press/v80/xiao18a">"Dynamical Isometry and a Mean Field Theory of CNNs: How to Train 10,000-Layer Vanilla Convolutional Neural Networks"</a>. <i>Proceedings of the 35th International Conference on Machine Learning</i>. PMLR: <span class="nowrap">5393–</span>5402. <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://arxiv.org/abs/1806.05393">1806.05393</a></span>.</cite></span>

</li>

<li id="cite_note-14"><span class="mw-cite-backlink"><b><a href="#cite_ref-14">^</a></b></span> <span class="reference-text"><cite id="CITEREFArjovskyShahBengio2016" class="citation journal cs1">Arjovsky, Martin; Shah, Amar; Bengio, Yoshua (2016-06-11). <a rel="nofollow" class="external text" href="https://proceedings.mlr.press/v48/arjovsky16.html">"Unitary Evolution Recurrent Neural Networks"</a>. <i>Proceedings of the 33rd International Conference on Machine Learning</i>. PMLR: <span class="nowrap">1120–</span>1128. <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://arxiv.org/abs/1511.06464">1511.06464</a></span>.</cite></span>

</li>

<li id="cite_note-15"><span class="mw-cite-backlink"><b><a href="#cite_ref-15">^</a></b></span> <span class="reference-text"><cite id="CITEREFHenaffSzlamLeCun2017" class="citation arxiv cs1">Henaff, Mikael; Szlam, Arthur; LeCun, Yann (2017-03-15). "Recurrent Orthogonal Networks and Long-Memory Tasks". <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://arxiv.org/abs/1602.06662">1602.06662</a></span> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/cs.NE">cs.NE</a>].</cite></span>

</li>

<li id="cite_note-16"><span class="mw-cite-backlink"><b><a href="#cite_ref-16">^</a></b></span> <span class="reference-text"><cite id="CITEREFMishkinMatas2016" class="citation cs2">Mishkin, Dmytro; Matas, Jiri (2016-02-19), <i>All you need is a good init</i>, <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://arxiv.org/abs/1511.06422">1511.06422</a></span></cite></span>

</li>

<li id="cite_note-17"><span class="mw-cite-backlink"><b><a href="#cite_ref-17">^</a></b></span> <span class="reference-text"><cite id="CITEREFXieXiongPu2017" class="citation conference cs1">Xie, Di; Xiong, Jiang; Pu, Shiliang (2017). <a rel="nofollow" class="external text" href="https://openaccess.thecvf.com/content_cvpr_2017/html/Xie_All_You_Need_CVPR_2017_paper.html"><i>All You Need Is Beyond a Good Init: Exploring Better Solution for Training Extremely Deep Convolutional Neural Networks With Orthonormality and Modulation</i></a>. IEEE Conference on Computer Vision and Pattern Recognition (CVPR). pp. <span class="nowrap">6176–</span>6185.</cite></span>

</li>

<li id="cite_note-18"><span class="mw-cite-backlink"><b><a href="#cite_ref-18">^</a></b></span> <span class="reference-text"><cite id="CITEREFZhangDauphinMa2019" class="citation arxiv cs1">Zhang, Hongyi; Dauphin, Yann N.; Ma, Tengyu (2019). "Fixup Initialization: Residual Learning Without Normalization". <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://arxiv.org/abs/1901.09321">1901.09321</a></span> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/cs.LG">cs.LG</a>].</cite></span>

</li>

<li id="cite_note-19"><span class="mw-cite-backlink"><b><a href="#cite_ref-19">^</a></b></span> <span class="reference-text"><cite id="CITEREFHuangPerezBaVolkovs2020" class="citation journal cs1">Huang, Xiao Shi; Perez, Felipe; Ba, Jimmy; Volkovs, Maksims (2020-11-21). <a rel="nofollow" class="external text" href="https://proceedings.mlr.press/v119/huang20f.html">"Improving Transformer Optimization Through Better Initialization"</a>. <i>Proceedings of the 37th International Conference on Machine Learning</i>. PMLR: <span class="nowrap">4475–</span>4483.</cite></span>

</li>

<li id="cite_note-:0-20"><span class="mw-cite-backlink">^ <a href="#cite_ref-:0_20-0"><sup><i><b>a</b></i></sup></a> <a href="#cite_ref-:0_20-1"><sup><i><b>b</b></i></sup></a></span> <span class="reference-text"><cite id="CITEREFMartens2010" class="citation journal cs1">Martens, James (2010-06-21). <a rel="nofollow" class="external text" href="https://dl.acm.org/doi/10.5555/3104322.3104416">"Deep learning via Hessian-free optimization"</a>. <i>Proceedings of the 27th International Conference on International Conference on Machine Learning</i>. ICML'10. Madison, WI, USA: Omnipress: <span class="nowrap">735–</span>742. <a href="ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <bdi>978-1-60558-907-7</bdi>.</cite></span>

</li>

<li id="cite_note-21"><span class="mw-cite-backlink"><b><a href="#cite_ref-21">^</a></b></span> <span class="reference-text"><cite id="CITEREFSussilloAbbott2014" class="citation arxiv cs1">Sussillo, David; Abbott, L. F. (2014). "Random Walk Initialization for Training Very Deep Feedforward Networks". <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://arxiv.org/abs/1412.6558">1412.6558</a></span> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/cs.NE">cs.NE</a>].</cite></span>

</li>

<li id="cite_note-22"><span class="mw-cite-backlink"><b><a href="#cite_ref-22">^</a></b></span> <span class="reference-text"><cite id="CITEREFBalduzziFreanLearyLewis2017" class="citation arxiv cs1">Balduzzi, David; Frean, Marcus; Leary, Lennox; Lewis, JP; Kurt Wan-Duo Ma; McWilliams, Brian (2017). "The Shattered Gradients Problem: If resnets are the answer, then what is the question?". <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://arxiv.org/abs/1702.08591">1702.08591</a></span> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/cs.NE">cs.NE</a>].</cite></span>

</li>

<li id="cite_note-23"><span class="mw-cite-backlink"><b><a href="#cite_ref-23">^</a></b></span> <span class="reference-text"><cite id="CITEREFLeCun1989" class="citation book cs1">LeCun, Y. (1989). <a rel="nofollow" class="external text" href="https://masters.donntu.ru/2012/fknt/umiarov/library/lecun.pdf">"Generalization and network design strategies"</a> <span class="cs1-format">(PDF)</span>. In Pfeifer, R.; Schreter, Z.; Fogelman, F.; Steels, L. (eds.). <i>Connectionism in Perspective: Proceedings of the International Conference Connectionism in Perspective, University of Zurich, 10–13 October 1988</i>. Amsterdam: Elsevier.</cite></span>

</li>

<li id="cite_note-24"><span class="mw-cite-backlink"><b><a href="#cite_ref-24">^</a></b></span> <span class="reference-text"><cite id="CITEREFKlambauerUnterthinerMayrHochreiter2017" class="citation journal cs1">Klambauer, Günter; Unterthiner, Thomas; Mayr, Andreas; Hochreiter, Sepp (2017). <a rel="nofollow" class="external text" href="https://proceedings.neurips.cc/paper_files/paper/2017/hash/5d44ee6f2c3f71b73125876103c8f6c4-Abstract.html">"Self-Normalizing Neural Networks"</a>. <i>Advances in Neural Information Processing Systems</i>. <b>30</b>. Curran Associates, Inc.</cite></span>

</li>

<li id="cite_note-25"><span class="mw-cite-backlink"><b><a href="#cite_ref-25">^</a></b></span> <span class="reference-text"><cite id="CITEREFBengio2009" class="citation journal cs1">Bengio, Y. (2009). <a rel="nofollow" class="external text" href="http://www.iro.umontreal.ca/~lisa/pointeurs/TR1312.pdf">"Learning Deep Architectures for AI"</a> <span class="cs1-format">(PDF)</span>. <i>Foundations and Trends in Machine Learning</i>. <b>2</b>: <span class="nowrap">1–</span>127. <a href="CiteSeerX_(identifier)" class="mw-redirect" title="CiteSeerX (identifier)">CiteSeerX</a> <span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.701.9550">10.1.1.701.9550</a></span>. <a href="Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1561%2F2200000006">10.1561/2200000006</a>.</cite></span>

</li>

<li id="cite_note-26"><span class="mw-cite-backlink"><b><a href="#cite_ref-26">^</a></b></span> <span class="reference-text"><cite id="CITEREFErhanCourvilleBengioVincent2010" class="citation journal cs1">Erhan, Dumitru; Courville, Aaron; Bengio, Yoshua; Vincent, Pascal (2010-03-31). <a rel="nofollow" class="external text" href="https://proceedings.mlr.press/v9/erhan10a.html">"Why Does Unsupervised Pre-training Help Deep Learning?"</a>. <i>Proceedings of the Thirteenth International Conference on Artificial Intelligence and Statistics</i>. JMLR Workshop and Conference Proceedings: <span class="nowrap">201–</span>208.</cite></span>

</li>

<li id="cite_note-27"><span class="mw-cite-backlink"><b><a href="#cite_ref-27">^</a></b></span> <span class="reference-text"><cite id="CITEREFBengioLamblinPopoviciLarochelle2006" class="citation journal cs1">Bengio, Yoshua; Lamblin, Pascal; Popovici, Dan; Larochelle, Hugo (2006). <a rel="nofollow" class="external text" href="https://proceedings.neurips.cc/paper/2006/hash/5da713a690c067105aeb2fae32403405-Abstract.html">"Greedy Layer-Wise Training of Deep Networks"</a>. <i>Advances in Neural Information Processing Systems</i>. <b>19</b>. MIT Press.</cite></span>

</li>

<li id="cite_note-28"><span class="mw-cite-backlink"><b><a href="#cite_ref-28">^</a></b></span> <span class="reference-text"><cite id="CITEREFGlorotBordesBengio2011" class="citation journal cs1">Glorot, Xavier; Bordes, Antoine; Bengio, Yoshua (2011-06-14). <a rel="nofollow" class="external text" href="https://proceedings.mlr.press/v15/glorot11a">"Deep Sparse Rectifier Neural Networks"</a>. <i>Proceedings of the Fourteenth International Conference on Artificial Intelligence and Statistics</i>. JMLR Workshop and Conference Proceedings: <span class="nowrap">315–</span>323.</cite></span>

</li>

<li id="cite_note-29"><span class="mw-cite-backlink"><b><a href="#cite_ref-29">^</a></b></span> <span class="reference-text"><cite id="CITEREFSutskeverMartensDahlHinton2013" class="citation journal cs1">Sutskever, Ilya; Martens, James; Dahl, George; Hinton, Geoffrey (2013-05-26). <a rel="nofollow" class="external text" href="https://www.cs.utoronto.ca/~ilya/pubs/2013/1051_2.pdf">"On the importance of initialization and momentum in deep learning"</a> <span class="cs1-format">(PDF)</span>. <i>Proceedings of the 30th International Conference on Machine Learning</i>. PMLR: <span class="nowrap">1139–</span>1147.</cite></span>

</li>

<li id="cite_note-30"><span class="mw-cite-backlink"><b><a href="#cite_ref-30">^</a></b></span> <span class="reference-text"><cite id="CITEREFBjorckGomesSelmanWeinberger2018" class="citation journal cs1">Bjorck, Nils; Gomes, Carla P; Selman, Bart; Weinberger, Kilian Q (2018). <a rel="nofollow" class="external text" href="https://proceedings.neurips.cc/paper/2018/hash/36072923bfc3cf47745d704feb489480-Abstract.html">"Understanding Batch Normalization"</a>. <i>Advances in Neural Information Processing Systems</i>. <b>31</b>. Curran Associates, Inc. <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://arxiv.org/abs/1806.02375">1806.02375</a></span>.</cite></span>

</li>

<li id="cite_note-31"><span class="mw-cite-backlink"><b><a href="#cite_ref-31">^</a></b></span> <span class="reference-text"><cite id="CITEREFBallesHennig2018" class="citation journal cs1">Balles, Lukas; Hennig, Philipp (2018-07-03). <a rel="nofollow" class="external text" href="https://proceedings.mlr.press/v80/balles18a">"Dissecting Adam: The Sign, Magnitude and Variance of Stochastic Gradients"</a>. <i>Proceedings of the 35th International Conference on Machine Learning</i>. PMLR: <span class="nowrap">404–</span>413. <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://arxiv.org/abs/1705.07774">1705.07774</a></span>.</cite></span>

</li>

<li id="cite_note-32"><span class="mw-cite-backlink"><b><a href="#cite_ref-32">^</a></b></span> <span class="reference-text"><cite id="CITEREFBrockDeSmithSimonyan2021" class="citation arxiv cs1">Brock, Andrew; De, Soham; Smith, Samuel L.; Simonyan, Karen (2021). "High-Performance Large-Scale Image Recognition Without Normalization". <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<span class="id-lock-free" title="Freely accessible"><a rel="nofollow" class="external text" href="https://arxiv.org/abs/2102.06171">2102.06171</a></span> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/cs.CV">cs.CV</a>].</cite></span>

</li>

</ol></div>

<div class="mw-heading mw-heading2"><h2 id="Further_reading">Further reading</h2></div>

<ul><li><cite id="CITEREFGoodfellowBengioCourville2016" class="citation book cs1">Goodfellow, Ian; Bengio, Yoshua; Courville, Aaron (2016). <a rel="nofollow" class="external text" href="https://www.deeplearningbook.org/contents/optimization.html">"8.4 Parameter Initialization Strategies"</a>. <i>Deep learning</i>. Adaptive computation and machine learning. Cambridge, Mass: The MIT press. <a href="ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <bdi>978-0-262-03561-3</bdi>.</cite></li>

<li><cite id="CITEREFNarkhedeBartakkeSutaone2021" class="citation journal cs1">Narkhede, Meenal V.; Bartakke, Prashant P.; Sutaone, Mukul S. (June 28, 2021). "A review on weight initialization strategies for neural networks". <i>Artificial Intelligence Review</i>. <b>55</b> (1). Springer Science and Business Media LLC: <span class="nowrap">291–</span>322. <a href="Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1007%2Fs10462-021-10033-z">10.1007/s10462-021-10033-z</a>. <a href="ISSN_(identifier)" class="mw-redirect" title="ISSN (identifier)">ISSN</a> <a rel="nofollow" class="external text" href="https://search.worldcat.org/issn/0269-2821">0269-2821</a>.</cite></li></ul>

/* start https://en.wikipedia.org/ */

.mw-parser-output .navbox{box-sizing:border-box;border:1px solid #a2a9b1;width:100%;clear:both;font-size:88%;text-align:center;padding:1px;margin:1em auto 0}.mw-parser-output .navbox .navbox{margin-top:0}.mw-parser-output .navbox+.navbox,.mw-parser-output .navbox+.navbox-styles+.navbox{margin-top:-1px}.mw-parser-output .navbox-inner,.mw-parser-output .navbox-subgroup{width:100%}.mw-parser-output .navbox-group,.mw-parser-output .navbox-title,.mw-parser-output .navbox-abovebelow{padding:0.25em 1em;line-height:1.5em;text-align:center}.mw-parser-output .navbox-group{white-space:nowrap;text-align:right}.mw-parser-output .navbox,.mw-parser-output .navbox-subgroup{background-color:#fdfdfd}.mw-parser-output .navbox-list{line-height:1.5em;border-color:#fdfdfd}.mw-parser-output .navbox-list-with-group{text-align:left;border-left-width:2px;border-left-style:solid}.mw-parser-output tr+tr>.navbox-abovebelow,.mw-parser-output tr+tr>.navbox-group,.mw-parser-output tr+tr>.navbox-image,.mw-parser-output tr+tr>.navbox-list{border-top:2px solid #fdfdfd}.mw-parser-output .navbox-title{background-color:#ccf}.mw-parser-output .navbox-abovebelow,.mw-parser-output .navbox-group,.mw-parser-output .navbox-subgroup .navbox-title{background-color:#ddf}.mw-parser-output .navbox-subgroup .navbox-group,.mw-parser-output .navbox-subgroup .navbox-abovebelow{background-color:#e6e6ff}.mw-parser-output .navbox-even{background-color:#f7f7f7}.mw-parser-output .navbox-odd{background-color:transparent}.mw-parser-output .navbox .hlist td dl,.mw-parser-output .navbox .hlist td ol,.mw-parser-output .navbox .hlist td ul,.mw-parser-output .navbox td.hlist dl,.mw-parser-output .navbox td.hlist ol,.mw-parser-output .navbox td.hlist ul{padding:0.125em 0}.mw-parser-output .navbox .navbar{display:block;font-size:100%}.mw-parser-output .navbox-title .navbar{float:left;text-align:left;margin-right:0.5em}body.skin--responsive .mw-parser-output .navbox-image img{max-width:none!important}@media print{body.ns-0 .mw-parser-output .navbox{display:none!important}}

/* end https://en.wikipedia.org/ */

</style></div><div role="navigation" class="navbox" aria-labelledby="Artificial_intelligence_(AI)426" style="padding:3px"><table class="nowraplinks hlist mw-collapsible autocollapse navbox-inner" style="border-spacing:0;background:transparent;color:inherit"><tbody><tr><th scope="col" class="navbox-title" colspan="2"><div id="Artificial_intelligence_(AI)426" style="font-size:114%;margin:0 4em"><a href="Artificial_intelligence" title="Artificial intelligence">Artificial intelligence</a> (AI)</div></th></tr><tr><td class="navbox-abovebelow" colspan="2"><div>

<ul><li><a href="History_of_artificial_intelligence" title="History of artificial intelligence">History</a>

<ul><li><a href="Timeline_of_artificial_intelligence" title="Timeline of artificial intelligence">timeline</a></li></ul></li>

<li><a href="List_of_artificial_intelligence_companies" title="List of artificial intelligence companies">Companies</a></li>

<li><a href="List_of_artificial_intelligence_projects" title="List of artificial intelligence projects">Projects</a></li></ul>

</div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%">Concepts</th><td class="navbox-list-with-group navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em">

<ul><li><a href="Parameter" title="Parameter">Parameter</a>

<ul><li><a href="Hyperparameter_(machine_learning)" title="Hyperparameter (machine learning)">Hyperparameter</a></li></ul></li>

<li><a href="Loss_functions_for_classification" title="Loss functions for classification">Loss functions</a></li>

<li><a href="Regression_analysis" title="Regression analysis">Regression</a>

<ul><li><a href="Bias%E2%80%93variance_tradeoff" title="Bias–variance tradeoff">Bias–variance tradeoff</a></li>

<li><a href="Double_descent" title="Double descent">Double descent</a></li>

<li><a href="Overfitting" title="Overfitting">Overfitting</a></li></ul></li>

<li><a href="Cluster_analysis" title="Cluster analysis">Clustering</a></li>

<li><a href="Gradient_descent" title="Gradient descent">Gradient descent</a>

<li><a href="Quasi-Newton_method" title="Quasi-Newton method">Quasi-Newton method</a></li>

<li><a href="Conjugate_gradient_method" title="Conjugate gradient method">Conjugate gradient method</a></li></ul></li>

<li><a href="Backpropagation" title="Backpropagation">Backpropagation</a></li>

<li><a href="Attention_(machine_learning)" title="Attention (machine learning)">Attention</a></li>

<li><a href="Convolution" title="Convolution">Convolution</a></li>

<li><a href="Normalization_(machine_learning)" title="Normalization (machine learning)">Normalization</a>

<ul><li><a href="Batch_normalization" title="Batch normalization">Batchnorm</a></li></ul></li>

<li><a href="Activation_function" title="Activation function">Activation</a>

<ul><li><a href="Softmax_function" title="Softmax function">Softmax</a></li>

<li><a href="Sigmoid_function" title="Sigmoid function">Sigmoid</a></li>

<li><a href="Rectifier_(neural_networks)" title="Rectifier (neural networks)">Rectifier</a></li></ul></li>

<li><a href="Gating_mechanism" title="Gating mechanism">Gating</a></li>

<li><a href="Regularization_(mathematics)" title="Regularization (mathematics)">Regularization</a></li>

<li><a href="Training%2C_validation%2C_and_test_data_sets" title="Training, validation, and test data sets">Datasets</a>

<ul><li><a href="Data_augmentation" title="Data augmentation">Augmentation</a></li></ul></li>

<li><a href="Prompt_engineering" title="Prompt engineering">Prompt engineering</a></li>

<li><a href="Reinforcement_learning" title="Reinforcement learning">Reinforcement learning</a>

<ul><li><a href="Q-learning" title="Q-learning">Q-learning</a></li>

<li><a href="State%E2%80%93action%E2%80%93reward%E2%80%93state%E2%80%93action" title="State–action–reward–state–action">SARSA</a></li>

<li><a href="Imitation_learning" title="Imitation learning">Imitation</a></li>

<li><a href="Policy_gradient_method" title="Policy gradient method">Policy gradient</a></li></ul></li>

<li><a href="Diffusion_process" title="Diffusion process">Diffusion</a></li>

<li><a href="Latent_diffusion_model" title="Latent diffusion model">Latent diffusion model</a></li>

<li><a href="Autoregressive_model" title="Autoregressive model">Autoregression</a></li>

<li><a href="Adversarial_machine_learning" title="Adversarial machine learning">Adversary</a></li>

<li><a href="Uncanny_valley" title="Uncanny valley">Uncanny valley</a></li>

<li><a href="Self-supervised_learning" title="Self-supervised learning">Self-supervised learning</a></li>

<li><a href="Reflection_(artificial_intelligence)" class="mw-redirect" title="Reflection (artificial intelligence)">Reflection</a></li>

<li><a href="Recursive_self-improvement" title="Recursive self-improvement">Recursive self-improvement</a></li>

<li><a href="Hallucination_(artificial_intelligence)" title="Hallucination (artificial intelligence)">Hallucination</a></li>

<li><a href="Word_embedding" title="Word embedding">Word embedding</a></li>

<li><a href="Vibe_coding" title="Vibe coding">Vibe coding</a></li></ul>

</div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%">Applications</th><td class="navbox-list-with-group navbox-list navbox-even" style="width:100%;padding:0"><div style="padding:0 0.25em">

<ul><li><a href="Machine_learning" title="Machine learning">Machine learning</a>

<ul><li><a href="Prompt_engineering#In-context_learning" title="Prompt engineering">In-context learning</a></li></ul></li>

<li><a href="Neural_network_(machine_learning)" title="Neural network (machine learning)">Artificial neural network</a>

<ul><li><a href="Deep_learning" title="Deep learning">Deep learning</a></li></ul></li>

<li><a href="Language_model" title="Language model">Language model</a>

<ul><li><a href="Large_language_model" title="Large language model">Large language model</a></li>

<li><a href="Reasoning_language_model" title="Reasoning language model">Reasoning language model</a></li>

<li><a href="Model_Context_Protocol" title="Model Context Protocol">Model Context Protocol</a></li>

<li><a href="Intelligent_agent" title="Intelligent agent">Intelligent agent</a></li>

<li><a href="Artificial_human_companion" title="Artificial human companion">Artificial human companion</a></li>

<li><a href="Humanity's_Last_Exam" title="Humanity's Last Exam">Humanity's Last Exam</a></li>

<li><a href="Artificial_general_intelligence" title="Artificial general intelligence">Artificial general intelligence (AGI)</a></li></ul>

</div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%">Implementations</th><td class="navbox-list-with-group navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em"></div><table class="nowraplinks navbox-subgroup" style="border-spacing:0"><tbody><tr><th scope="row" class="navbox-group" style="width:1%">Audio–visual</th><td class="navbox-list-with-group navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em">

<ul><li><a href="AlexNet" title="AlexNet">AlexNet</a></li>

<li><a href="WaveNet" title="WaveNet">WaveNet</a></li>

<li><a href="Human_image_synthesis" title="Human image synthesis">Human image synthesis</a></li>

<li><a href="Computer_vision" title="Computer vision">Computer vision</a></li>

<li><a href="Deep_learning_speech_synthesis" title="Deep learning speech synthesis">Speech synthesis</a>

<li><a href="ElevenLabs" title="ElevenLabs">ElevenLabs</a></li></ul></li>

<li><a href="Speech_recognition" title="Speech recognition">Speech recognition</a>

<ul><li><a href="Whisper_(speech_recognition_system)" title="Whisper (speech recognition system)">Whisper</a></li></ul></li>

<li><a href="Facial_recognition_system" title="Facial recognition system">Facial recognition</a></li>

<li><a href="AlphaFold" title="AlphaFold">AlphaFold</a></li>

<li><a href="Text-to-image_model" title="Text-to-image model">Text-to-image models</a>

<ul><li><a href="Aurora_(text-to-image_model)" class="mw-redirect" title="Aurora (text-to-image model)">Aurora</a></li>

<li><a href="Adobe_Firefly" title="Adobe Firefly">Firefly</a></li>

<li><a href="Ideogram_(text-to-image_model)" title="Ideogram (text-to-image model)">Ideogram</a></li>

<li><a href="Imagen_(text-to-image_model)" title="Imagen (text-to-image model)">Imagen</a></li>

<li><a href="Midjourney" title="Midjourney">Midjourney</a></li>

<li><a href="Recraft" title="Recraft">Recraft</a></li>

<li><a href="Stable_Diffusion" title="Stable Diffusion">Stable Diffusion</a></li></ul></li>

<li><a href="Text-to-video_model" title="Text-to-video model">Text-to-video models</a>

<ul><li><a href="Dream_Machine_(text-to-video_model)" title="Dream Machine (text-to-video model)">Dream Machine</a></li>

<li><a href="Runway_(company)#Services_and_technologies" title="Runway (company)">Runway Gen</a></li>

<li><a href="MiniMax_(company)#Hailuo_AI" title="MiniMax (company)">Hailuo AI</a></li>

<li><a href="Kling_(text-to-video_model)" class="mw-redirect" title="Kling (text-to-video model)">Kling</a></li>

<li><a href="Music_and_artificial_intelligence" title="Music and artificial intelligence">Music generation</a>

<ul><li><a href="Riffusion" title="Riffusion">Riffusion</a></li>

</div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%">Text</th><td class="navbox-list-with-group navbox-list navbox-even" style="width:100%;padding:0"><div style="padding:0 0.25em">

<li><a href="GloVe" title="GloVe">GloVe</a></li>

<li><a href="Llama_(language_model)" title="Llama (language model)">Llama</a></li>

<li><a href="Chinchilla_(language_model)" title="Chinchilla (language model)">Chinchilla AI</a></li>

<li><a href="ChatGPT" title="ChatGPT">ChatGPT</a></li>

<li><a href="Claude_(language_model)" title="Claude (language model)">Claude</a></li>

<li><a href="Gemini_(language_model)" title="Gemini (language model)">Gemini</a>

<ul><li><a href="Gemini_(chatbot)" title="Gemini (chatbot)">chatbot</a></li></ul></li>

<li><a href="LaMDA" title="LaMDA">LaMDA</a></li>

<li><a href="BLOOM_(language_model)" title="BLOOM (language model)">BLOOM</a></li>

<li><a href="Project_Debater" title="Project Debater">Project Debater</a></li>

<li><a href="IBM_Watson" title="IBM Watson">IBM Watson</a></li>

<li><a href="IBM_Watsonx" title="IBM Watsonx">IBM Watsonx</a></li>

<li><a href="IBM_Granite" title="IBM Granite">Granite</a></li>

<li><a href="Huawei_PanGu" title="Huawei PanGu">PanGu-Σ</a></li>

<li><a href="DeepSeek_(chatbot)" title="DeepSeek (chatbot)">DeepSeek</a></li>

</div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%">Decisional</th><td class="navbox-list-with-group navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em">

<ul><li><a href="AlphaGo" title="AlphaGo">AlphaGo</a></li>

<li><a href="AlphaZero" title="AlphaZero">AlphaZero</a></li>

<li><a href="OpenAI_Five" title="OpenAI Five">OpenAI Five</a></li>

<li><a href="Self-driving_car" title="Self-driving car">Self-driving car</a></li>

<li><a href="MuZero" title="MuZero">MuZero</a></li>

<li><a href="Action_selection" title="Action selection">Action selection</a>

<ul><li><a href="AutoGPT" title="AutoGPT">AutoGPT</a></li></ul></li>

<li><a href="Robot_control" title="Robot control">Robot control</a></li></ul>

</div></td></tr></tbody></table><div></div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%">People</th><td class="navbox-list-with-group navbox-list navbox-even" style="width:100%;padding:0"><div style="padding:0 0.25em">

<ul><li><a href="Alan_Turing" title="Alan Turing">Alan Turing</a></li>

<li><a href="Warren_Sturgis_McCulloch" title="Warren Sturgis McCulloch">Warren Sturgis McCulloch</a></li>

<li><a href="Walter_Pitts" title="Walter Pitts">Walter Pitts</a></li>

<li><a href="John_von_Neumann" title="John von Neumann">John von Neumann</a></li>

<li><a href="Claude_Shannon" title="Claude Shannon">Claude Shannon</a></li>

<li><a href="Shun'ichi_Amari" title="Shun'ichi Amari">Shun'ichi Amari</a></li>

<li><a href="Kunihiko_Fukushima" title="Kunihiko Fukushima">Kunihiko Fukushima</a></li>

<li><a href="Takeo_Kanade" title="Takeo Kanade">Takeo Kanade</a></li>

<li><a href="Marvin_Minsky" title="Marvin Minsky">Marvin Minsky</a></li>

<li><a href="John_McCarthy_(computer_scientist)" title="John McCarthy (computer scientist)">John McCarthy</a></li>

<li><a href="Nathaniel_Rochester_(computer_scientist)" title="Nathaniel Rochester (computer scientist)">Nathaniel Rochester</a></li>

<li><a href="Allen_Newell" title="Allen Newell">Allen Newell</a></li>

<li><a href="Cliff_Shaw" title="Cliff Shaw">Cliff Shaw</a></li>

<li><a href="Herbert_A._Simon" title="Herbert A. Simon">Herbert A. Simon</a></li>

<li><a href="Oliver_Selfridge" title="Oliver Selfridge">Oliver Selfridge</a></li>

<li><a href="Frank_Rosenblatt" title="Frank Rosenblatt">Frank Rosenblatt</a></li>

<li><a href="Bernard_Widrow" title="Bernard Widrow">Bernard Widrow</a></li>

<li><a href="Joseph_Weizenbaum" title="Joseph Weizenbaum">Joseph Weizenbaum</a></li>

<li><a href="Seymour_Papert" title="Seymour Papert">Seymour Papert</a></li>

<li><a href="Seppo_Linnainmaa" title="Seppo Linnainmaa">Seppo Linnainmaa</a></li>

<li><a href="Paul_Werbos" title="Paul Werbos">Paul Werbos</a></li>

<li><a href="Geoffrey_Hinton" title="Geoffrey Hinton">Geoffrey Hinton</a></li>

<li><a href="John_Hopfield" title="John Hopfield">John Hopfield</a></li>

<li><a href="J%C3%BCrgen_Schmidhuber" title="Jürgen Schmidhuber">Jürgen Schmidhuber</a></li>

<li><a href="Yann_LeCun" title="Yann LeCun">Yann LeCun</a></li>

<li><a href="Yoshua_Bengio" title="Yoshua Bengio">Yoshua Bengio</a></li>

<li><a href="Lotfi_A._Zadeh" title="Lotfi A. Zadeh">Lotfi A. Zadeh</a></li>

<li><a href="Stephen_Grossberg" title="Stephen Grossberg">Stephen Grossberg</a></li>

<li><a href="Alex_Graves_(computer_scientist)" title="Alex Graves (computer scientist)">Alex Graves</a></li>

<li><a href="James_Goodnight" title="James Goodnight">James Goodnight</a></li>

<li><a href="Andrew_Ng" title="Andrew Ng">Andrew Ng</a></li>

<li><a href="Ilya_Sutskever" title="Ilya Sutskever">Ilya Sutskever</a></li>

<li><a href="Alex_Krizhevsky" title="Alex Krizhevsky">Alex Krizhevsky</a></li>

<li><a href="Ian_Goodfellow" title="Ian Goodfellow">Ian Goodfellow</a></li>

<li><a href="Demis_Hassabis" title="Demis Hassabis">Demis Hassabis</a></li>

<li><a href="David_Silver_(computer_scientist)" title="David Silver (computer scientist)">David Silver</a></li>

<li><a href="Andrej_Karpathy" title="Andrej Karpathy">Andrej Karpathy</a></li>

<li><a href="Ashish_Vaswani" title="Ashish Vaswani">Ashish Vaswani</a></li>

<li><a href="Noam_Shazeer" title="Noam Shazeer">Noam Shazeer</a></li>

<li><a href="Aidan_Gomez" title="Aidan Gomez">Aidan Gomez</a></li>

<li><a href="Mustafa_Suleyman" title="Mustafa Suleyman">Mustafa Suleyman</a></li>

<li><a href="Fran%C3%A7ois_Chollet" title="François Chollet">François Chollet</a></li></ul>

</div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%">Architectures</th><td class="navbox-list-with-group navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em">

<ul><li><a href="Neural_Turing_machine" title="Neural Turing machine">Neural Turing machine</a></li>

<li><a href="Differentiable_neural_computer" title="Differentiable neural computer">Differentiable neural computer</a></li>

<li><a href="Transformer_(deep_learning_architecture)" title="Transformer (deep learning architecture)">Transformer</a>

<ul><li><a href="Vision_transformer" title="Vision transformer">Vision transformer (ViT)</a></li></ul></li>

<li><a href="Recurrent_neural_network" title="Recurrent neural network">Recurrent neural network (RNN)</a></li>

<li><a href="Long_short-term_memory" title="Long short-term memory">Long short-term memory (LSTM)</a></li>

<li><a href="Gated_recurrent_unit" title="Gated recurrent unit">Gated recurrent unit (GRU)</a></li>

<li><a href="Echo_state_network" title="Echo state network">Echo state network</a></li>

<li><a href="Multilayer_perceptron" title="Multilayer perceptron">Multilayer perceptron (MLP)</a></li>

<li><a href="Convolutional_neural_network" title="Convolutional neural network">Convolutional neural network (CNN)</a></li>

<li><a href="Residual_neural_network" title="Residual neural network">Residual neural network (RNN)</a></li>

<li><a href="Highway_network" title="Highway network">Highway network</a></li>

<li><a href="Mamba_(deep_learning_architecture)" title="Mamba (deep learning architecture)">Mamba</a></li>

<li><a href="Autoencoder" title="Autoencoder">Autoencoder</a></li>

<li><a href="Variational_autoencoder" title="Variational autoencoder">Variational autoencoder (VAE)</a></li>

<li><a href="Generative_adversarial_network" title="Generative adversarial network">Generative adversarial network (GAN)</a></li>

<li><a href="Graph_neural_network" title="Graph neural network">Graph neural network (GNN)</a></li></ul>

</div></td></tr><tr><td class="navbox-abovebelow" colspan="2"><div>

<ul><li><span class="noviewer" typeof="mw:File"><span title="Category"></span></span> Category</li></ul>

</div></td></tr></tbody></table></div></div><div><div class="zim-footer">

This article is issued from <a class="external text" title="Last edited on 2025-06-20" href="https://en.wikipedia.org/wiki/?title=Weight_initialization&oldid=1296589182">Wikipedia</a>. The text is available under <a class="external text" href="https://creativecommons.org/licenses/by-sa/4.0/deed.en">Creative Commons Attribution-Share Alike 4.0</a> unless otherwise noted. Additional terms may apply for the media files.

</div>

</div></div>

</div>

</main>

</div>

</body></html>